From 0aa0be5542d1982a3241cc93a718cf9cea549789 Mon Sep 17 00:00:00 2001 From: Benjamin Diedrichsen Date: Fri, 31 Jul 2026 18:21:43 +0200 Subject: [PATCH] hardening and bugfixing prior to stable release --- .gitignore | 3 + DOCS-AUDIT.md | 175 ++++- FIELD-PLAN.md | 742 ++++++++++++++++++ README.md | 19 +- packages/nopy-cubes-core/README.md | 11 +- .../cubes/runtime/docker/README.md | 4 +- .../cubes/runtime/nodevm/README.md | 106 +-- .../cubes/runtime/nodevm/deploy.py | 73 +- .../cubes/runtime/nodevm/manifest.mjs | 14 +- .../nopy-cubes-core/cubes/user/add/README.md | 53 +- packages/nopy-cubes/src/types.ts | 12 + packages/nopy/README.md | 118 ++- packages/nopy/docs/API.md | 100 ++- packages/nopy/docs/CUBE-PACKAGES.md | 4 +- packages/nopy/docs/FACTS.md | 348 ++++++++ packages/nopy/docs/SESSION_FORMAT.md | 41 +- packages/nopy/docs/VAGRANT.md | 37 +- packages/nopy/example.nopysession.json | 2 + packages/nopy/src/cubes/dependencies.ts | 82 +- packages/nopy/src/nopy.cli.ts | 43 +- packages/nopy/src/nopy.common.ts | 64 +- packages/nopy/src/nopy.config.ts | 13 +- packages/nopy/src/nopy.errors.ts | 50 ++ packages/nopy/src/nopy.executor.ts | 14 +- packages/nopy/src/nopy.exit.ts | 2 +- packages/nopy/src/nopy.history.ts | 33 +- packages/nopy/src/nopy.main.ts | 70 +- packages/nopy/src/nopy.prompts.ts | 76 +- packages/nopy/src/nopy.session.ts | 92 ++- packages/nopy/src/nopy.workflow.ts | 20 +- packages/nopy/tests/common.test.ts | 63 ++ .../tests/cubes.dependencies.edge.test.ts | 123 ++- .../nopy/tests/cubes.dependencies.test.ts | 50 ++ packages/nopy/tests/errors.test.ts | 62 ++ packages/nopy/tests/executor.test.ts | 14 - packages/nopy/tests/fixtures/form-probe.ts | 42 + packages/nopy/tests/main.test.ts | 160 ++-- packages/nopy/tests/prompts.pty.test.ts | 77 ++ packages/nopy/tests/prompts.test.ts | 92 ++- packages/nopy/tests/session.test.ts | 119 ++- packages/nopy/tests/workflow.test.ts | 12 +- scripts/drive.py | 57 ++ scripts/expect.py | 110 +++ scripts/ptysize.py | 50 ++ 44 files changed, 3016 insertions(+), 436 deletions(-) create mode 100644 FIELD-PLAN.md create mode 100644 packages/nopy/docs/FACTS.md create mode 100644 packages/nopy/src/nopy.errors.ts create mode 100644 packages/nopy/tests/errors.test.ts create mode 100644 packages/nopy/tests/fixtures/form-probe.ts create mode 100644 packages/nopy/tests/prompts.pty.test.ts create mode 100755 scripts/drive.py create mode 100755 scripts/expect.py create mode 100644 scripts/ptysize.py diff --git a/.gitignore b/.gitignore index 6d13610..9730ec1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # anyway (see the Vagrantfile). .vagrant-hostkeys .python-version +# The pty drivers under scripts/ import each other, so running one leaves a +# bytecode cache next to them. +__pycache__/ node_modules cache diff --git a/DOCS-AUDIT.md b/DOCS-AUDIT.md index efd0a40..9d0f279 100644 --- a/DOCS-AUDIT.md +++ b/DOCS-AUDIT.md @@ -19,17 +19,41 @@ the record of what was wrong. So far: §1.1 (`--use-defaults`), §2.2 (`getDefaults()`), §2.1 (precedence — the second half closed differently than proposed), §3 in full (`docs/API.md`, regenerated), §4.2 (password on stdout — points 1 and 2 of 3), §4.3 (what a session records), §2.9 (the nopy README's -yarn install instructions), and one bullet of §6.4. +yarn install instructions), §2.4 (`version` / `timestamp`, implemented rather +than deleted), §2.5 (`listSessions`' filename filter), §4.5 (`-s` on a replay, +plus `-l` and history), §6.7 (a secret written to the session in plaintext), and +one bullet of §6.4. Closing §3 also settled the documentation half of several findings elsewhere -without touching their underlying cause: §1.2, §1.3, §1.5, §2.3, §2.7, §4.4 and +without touching their underlying cause: §1.3, §1.5, §2.3, §2.7, §4.4 and §6.5 are each now stated accurately in `docs/API.md`, but the code still behaves -as those findings describe and they stay open. +as those findings describe and they stay open. §1.2 was closed outright by +removing the flag. + +## Where the drift is + +A field run against a fresh VM sorted the findings for us, and they cluster on +one seam. **Everything a human reads on screen matched the documentation. +Everything machine-facing had drifted** — `--json`, the session format, `-s` on a +replay, `-l` and history, `runtime:nodevm`'s parameters, the bundle's install +command. + +That is not random rot. The interactive surface is maintained by daily use: a +wrong prompt label is noticed the next time someone runs the thing. The scripting +surface was documented from intent and then never exercised, so nothing pushed +back when it changed or was never built. + +Corrections come in the two shapes that distinction implies. `--json` was +documented from intent and never built, so it was removed (§1.2). The rest was +built and then drifted, so it was fixed. Keeping it fixed means what remains of +the scripting surface — `--print-only`, sessions, history — needs tests that +assert on **stdout**, not prose. --- ## Contents +- [Where the drift is](#where-the-drift-is) - [1. Documented features that do not exist](#1-documented-features-that-do-not-exist) - [2. Documented behaviour that differs from the code](#2-documented-behaviour-that-differs-from-the-code) - [3. ✅ `docs/API.md` — systematic drift — fixed](#3--docsapimd--systematic-drift--fixed) @@ -82,7 +106,7 @@ nopy.cli.ts:95 useDefaults: options... # passed cubes/dependencies.ts:35 useDefaults?: boolean; # declared — and that is all ``` -### 1.2 🔴 `-j, --json` produces no output on success +### 1.2 🔴 `-j, --json` produces no output on success — **closed by removal** | | | |---|---| @@ -104,6 +128,17 @@ Related: `--dry-run --json` prints the **text** plan, not JSON. argument (`nopy.executor.ts:172`), even though the function supports it (`nopy.executor.ts:110`). +**Closed by deleting the flag, not by implementing it.** `executeDeployCalls` +runs pyinfra with *inherited* stdio, so during a run nopy does not own its own +stdout — pyinfra does. A JSON blob appended after an unbounded amount of another +process's output is not machine-readable by any definition a caller could rely +on; making it so means capturing pyinfra's output and giving up live progress. +Same root cause as `ExecutionResult.stdout` never being populated. What replaced +it is a promise a test can hold to: **stdout carries the deploy commands and +pyinfra's own output, everything nopy says about itself goes to stderr**, and the +exit code is the verdict. `nopy history --json` is a different flag, it works, +and it stays. + ### 1.3 🟠 `log.verbosity` and `log.debug` have no effect Pre-existing known drift, recorded in `CLAUDE.md`, but the README still presents @@ -275,7 +310,19 @@ the two documents disagree, and neither mentions that it matters. `net:tailscale` (all 4 fields), `runtime:nodevm` (all 4), `user:add` (all 4), `ssh:keygen` (all 4) and `admin:locale` (all 4). -### 2.4 🟠 Session files claim `version` and `timestamp` fields +### 2.4 ✅ Session files claim `version` and `timestamp` fields — **fixed** + +Closed by implementing them rather than deleting the claim. `createSession` +stamps `version: '1.0.0'` (exported as `SESSION_VERSION`) and an ISO +`timestamp`; `nopy()` fills in a default `name` at save time, using the same +`describeSession()` the history list uses — one implementation, so the two +cannot drift. `loadSession` still requires only `cubes` and `auth`, so every +session written before this, and every hand-written one, keeps loading; an +unrecognised `version` is a warning on stderr, never a refusal. The interface +and both documents now mark the three fields optional, which is what they are. + +The original finding follows. + `README.md:179-181` shows a session with `"version": "1.0.0"` and `"timestamp": "2025-10-13T10:30:00Z"`, and `docs/SESSION_FORMAT.md:305-306` @@ -293,7 +340,14 @@ Consequence: a `version` field implies a compatibility check that does not exist Nothing reads it, so an incompatible old session fails later and more obscurely than a version check would. -### 2.5 🟠 Session filename convention does not match `listSessions()` +### 2.5 ✅ Session filename convention does not match `listSessions()` — **fixed** + +`listSessions` now matches `*.nopysession.json` and `*.nopysession.mjs` as well +as the two shorter suffixes — `saveSession` writes whatever path it is handed, +so files under the old name exist and there was no reason to stop finding them. + +The original finding follows. + The READMEs consistently use `*.nopysession.json` (`README.md:223`, `330`, `338`; `docs/DOCKER.md:54`; the shipped `example.nopysession.json`). @@ -443,8 +497,7 @@ first. > > Three things were deliberately added rather than merely corrected. A > **Known gaps** section states the behaviour a reader would otherwise take on -> trust — `logConfigToFlags` being unconsumed (§1.3), `--json` printing nothing -> on success (§1.2), the absent cycle detection (§1.5, §6.5), `DeployCall.dependencies` +> trust — `logConfigToFlags` being unconsumed (§1.3), the absent cycle detection (§1.5, §6.5), `DeployCall.dependencies` > always being `[]`, `ExecutionResult.stdout`/`stderr` never being populated, and > hook variables not being schema-validated (§2.7). The `.describe()`/`.default()` > ordering hazard (§2.3) is called out where the manifest example lives, with the @@ -633,7 +686,22 @@ Worth documenting alongside `--dry-run`, since the difference is not obvious: `--print-only` returns a `NopyResult` with `successful: 0` and skips execution entirely, while `--dry-run` goes through the executor. -### 4.5 🟡 `--save-session` is ignored during a replay +### 4.5 ✅ `--save-session` is ignored during a replay — **fixed** + +The guard is gone: the resolved cube set is exactly what the user asked to +capture, and a session written from a replay is no less valid than one written +from a fresh run. The README's "Recording a Session" examples now include the +replay form. + +Fixed alongside it, from the same field run: a `--load-session` run was excluded +from history along with `-R`/`-H`, which was right for the latter two and wrong +for the first — a session file has never been in history, so `nopy history` +reported nothing afterwards and `-R` had nothing to repeat. `WorkflowResult` +now carries `replaySource: 'file' | 'history' | undefined` instead of a boolean, +which is the distinction the boolean could not express. + +The original finding follows. + `nopy.main.ts:191` guards with `saveSessionPath && !workflow.isReplay`, so `nopy install -R -s out.json` writes nothing and says nothing. The @@ -806,6 +874,88 @@ same shape as the `PASSWORD` default that was removed. It is recorded in the session, so replays are stable, but each fresh `-D` run still creates a differently-named account. +### 6.7 ✅ A declared secret was written to the session file in plaintext — **fixed** + +Found in the acceptance run, not by reading. `README.md` promises that a session +holds no secret: "Passwords are never stored in session files. This covers both +the SSH password ... and any schema key a cube's manifest lists under `secrets`." +The `variables` block honoured that — `persistable()` leaves a declared secret +out entirely. The `env` block, one key higher in the same file, was a verbatim +copy of `.nopyrc.json`'s, so a credential declared there was written to the +session **and** to `.nopy.history.json` in plaintext. + +Two of the fixes above widened the blast radius before it was noticed: §4.5 made +`--save-session` work on a replay, and §3.2 started recording `--load-session` +runs to history. Both write more files than before. + +`Variables.persistableEnv()` applies the same rule to `env` that `persistable()` +applies to `variables`, and `nopy()` uses it instead of `config.env`. Verified in +the field: with `SSH_PASSWORD` declared under `secrets` and set in `env`, neither +the written session nor the history file contains the value. + +### 6.8 ✅ `--print-only` was recorded in history — **fixed** + +Also found in the acceptance run. `--dry-run` is excluded from history because it +deploys nothing; `--print-only`, which also deploys nothing, was not. Four +interactive runs against the VM produced four history entries, two of them from +`-P` passes that had only printed a command — and since `-R` repeats the head of +the list, the safe look-before-you-leap flag displaced the last real deployment +as the thing a bare `-R` would re-run. + +One condition, `!printOnly`, alongside the `!dryRun` it belongs with. The +`README` list of "a run is *not* recorded when" and `docs/API.md` say so now. + +### 6.9 ✅ Deploy order is the dependency tree; `CubeSelection` decides only the ties — **write-up corrected** + +Found in the acceptance run, and the first write-up of it here was wrong. It +claimed a fix "has to decide what the right order even is — the order they were +picked in, or a topological one over `dependencies()`". Neither: the order is the +dependency tree, and that is already what nopy does. `resolveCube` resolves +`dependencies()` before emitting the cube itself, so emission is DFS post-order — +a topological order by construction. The recursion *is* the sort, which is what +`docs/API.md` means by "no separate topological sort", and `nopy.main.ts` walking +`selectedCubes` cannot break it: a cube listed ahead of its own dependency still +drags that dependency in first, and the second visit is deduped by `callKey` +rather than re-emitted at the tail. Pinned by *deploy order across several +selected cubes* in `tests/cubes.dependencies.test.ts`, both ways round. + +This does not reopen §1.5, which stands: the *output* is a topological order but +there is no sort *algorithm*, and the price of that is still no cycle detection — +two mutually dependent cubes recurse until the stack overflows. + +What list order does decide is where a cube with **no** edge lands, and that is +the whole of the real finding. `CubeSelection` returns enquirer's `selected`, +which is `choices.filter(enabled)` — display order, sorted by cube id, not the +order you ticked them. So picking `user:add` and `runtime:nodevm` yields +`['runtime:nodevm', 'user:add']`, and nothing reorders them because +`runtime:nodevm` declares `dependencies: () => []`. The acceptance run split them +into two invocations. + +That missing edge is deliberate and stays missing: `user:add` *creates* a user, +so declaring it would make installing Node into an existing account silently +provision a new one. The prerequisite `SHELL=fish` really has is "fish and Oh My +Fish exist for `USER`", which no cube offers on its own — `user:add` only +provides it in passing. §5.3's `DeployError` is the answer for that, and it fires +before anything is changed. Ordering cannot fix an edge nobody can honestly +declare. + +One residue, verified and left alone: an `after` hook's `exec(id)` runs after its +own cube is emitted, so it expresses "B after A" — but if B is also selected and +listed first, B is emitted first and the intent inverts. `after` hooks are not +the dependency graph and no cube in the bundle relies on this. + +### 6.10 ✅ `runtime:nodevm` installed apt packages without refreshing the index — **fixed** + +The same defect as §5.3 one operation earlier, and it only surfaced once §5.3 was +fixed and the cube could be run on a box where nothing else had. `apt.packages` +was called without `update`, alone among the six cubes in the bundle that install +packages. On a fresh `bento/ubuntu-24.04` the shipped index names .deb versions +the mirror has already superseded, so the fetch 404s and pyinfra reports +`executed 0 commands` before nvm is ever reached. + +It passed on the first VM only because `user:add` had run there and pulled in +`apt:essentials`, which does pass `update`. Fixed with `update=True` on the call. + --- ## 7. Checked and accurate @@ -851,9 +1001,10 @@ Recording what was verified and found correct, so a future pass need not redo it **1 — ~~Decide on the three phantom features.~~ Two left.** §1.1 (`-D`) is **done** — implemented, tested, and verified against every cube in `cubes/`. That closed §2.2 and half of §2.1 with it, since neither could be left standing -under a run that never prompts. §1.2 (`--json`) and §1.3 (`log.*`) are still -"documented, wired up, never read": each is a small implementation or a small -deletion, but neither can stay documented as working. +under a run that never prompts. §1.2 (`--json`) is **done** — removed rather +than implemented, for the reason recorded there. §1.3 (`log.*`) is still +"documented, wired up, never read": a small implementation or a small deletion, +but it cannot stay documented as working. **4 — Decide the `.describe()`/`.default()` ordering (§2.3).** Either read through the `ZodDefault` wrapper in `nopy.prompts.ts`, or fix the ordering in all diff --git a/FIELD-PLAN.md b/FIELD-PLAN.md new file mode 100644 index 0000000..40ee4dd --- /dev/null +++ b/FIELD-PLAN.md @@ -0,0 +1,742 @@ +# Field-report implementation plan + +Turns the findings of the wild-run field report into work. Ordered by severity, +then by whether a fix unblocks a later one. Every phase is independently +shippable and ends at the existing gate (`lint:ci` → `typecheck` → +`test:coverage`). + +Findings the field run confirmed that `DOCS-AUDIT.md` already tracks keep their +audit number, so the two documents stay in step: closing an item here closes it +there. + +## Contents + +- [0. Retractions](#0-retractions) — two findings were harness artefacts +- [1. The secret leak](#1-the-secret-leak) — `env` broadcasts a credential in the clear +- [2. Remove `--json`](#2-remove---json) — audit §1.2, closed by deletion +- [3. Replay and session correctness](#3-replay-and-session-correctness) — audit §2.4, §2.5, §4.5 +- [4. The first five minutes](#4-the-first-five-minutes) +- [5. Cube defects](#5-cube-defects) — audit §5.3 +- [6. Documentation sweep](#6-documentation-sweep) +- [7. Harness fix and acceptance run](#7-harness-fix-and-acceptance-run) + +--- + +## 0. Retractions + +Two findings in the field report were caused by the PTY driver I used to script +the TUI, not by nopy. The driver called `pty.fork()` and never issued +`TIOCSWINSZ`, so the child saw a **0×0 terminal**. + +`enquirer`'s `utils.height` (`lib/utils.js:80-86`) computes a sane fallback and +then throws it away: + +```js +let rows = (stream && stream.rows) ? stream.rows : fallback; // fallback = 25 +if (stream && typeof stream.getWindowSize === 'function') { + rows = stream.getWindowSize()[1]; // ← unconditional +} +``` + +A TTY always has `getWindowSize`, so `height` becomes `0`, and +`ArrayPrompt.limit` (`lib/types/array.js:604`) returns `Math.min(limit, 0)`. +`visible` is then empty for every array prompt. + +Re-run with a 50×200 window, both work correctly: + +| Field report | Actual | +| --- | --- | +| §3.6 multi-field forms never render their fields | All four fields render, accept input, and submit: `RESULT {"USER":"X","PASSWORD":"changeme","GROUPS":"","PUBKEY":""}` | +| §3.14 cube filter says "No matching choices" while matching fine | Filter renders correctly, highlights the matched substring, and returns `["user:add"]` | + +**What survives, and it is worth fixing.** nopy has no defence against a +terminal that reports a degenerate size: the form silently submits `{}`, and the +run proceeds with every variable absent. That is [§4.4](#44-survive-a-terminal-that-reports-no-size) +and [§4.5](#45-never-deploy-a-cube-with-a-missing-required-variable). Field +report §3.7 (a required key dropped from the command) was reached through the +0×0 form, but the hole it exposed is real and independent: nothing on the +interactive path checks that a cube's required variables were actually filled. + +Everything else in the field report stands. + +--- + +## 1. The secret leak + +Highest severity: following the documentation as written prints a credential in +plaintext, and the workaround it is prescribed for does not work either. + +### 1.1 A declared secret must never be broadcast to cubes that do not declare it + +**What happens.** `Variables.bucket()` (`nopy.common.ts:193-203`) seeds *every* +key of config `env` onto *every* cube as an `env`-origin assignment, and +`isSecret` (`nopy.common.ts:135-137`) is keyed per cube. So with `PASSWORD` under +`env`, a dry run prints: + +``` +Step 1: apt:essentials … --data "PASSWORD=wildpass123" ← unmasked +Step 2: user:add … --data "PASSWORD=********" ← masked +Step 3: runtime:nodevm … --data "PASSWORD=wildpass123" ← unmasked +``` + +**Why the obvious fix is wrong.** "Seed `env` only onto cubes whose schema +declares the key" breaks a shipped cube: `ssh/keyman/deploy.py:28` reads +`host.data.get('KEY_DIR')`, a key its manifest does not declare and that exists +only in `.nopyrc.json` `env` (`packages/nopy/.nopyrc.json:5`). Broadcast is +load-bearing. + +**Fix.** Narrow the rule to secrets only — broadcast stays, secrets stop +travelling: + +1. In `nopy.main.ts`, after `loadCubes()`, collect the union of every loaded + manifest's `secrets` and hand it to `Variables`: + + ```ts + const declaredSecrets = new Set(Object.values(cubes).flatMap((c) => c.secrets)); + const variables = new Variables(config.env, declaredSecrets); + ``` + + Deterministic and ordering-free: it is computed before the first + `resolveCube`, so it does not depend on which cube resolves first. + +2. In `bucket()`, skip seeding an `env` key that is in `declaredSecrets` unless + the cube itself declares that key in its schema. `Variables` needs the cube's + schema keys for this — add `declareSchema(cube, keys)`, called from + `BuildContext.resolveCube` immediately after `declareSecrets` + (`cubes/dependencies.ts:121`), before any assignment creates the bucket. + +3. Mark globally, mask globally: a key in `declaredSecrets` is `redacted` on + whichever cube it does land on, even if that cube's own manifest forgot to + list it. Cheap defence against a manifest that declares `PASSWORD` in `schema` + and omits it from `secrets`. + +4. New optional `.nopyrc.json` key, for an `env` secret no manifest declares + (an API token a hook uses, say): + + ```json + { "secrets": ["DEPLOY_TOKEN"], "env": { "DEPLOY_TOKEN": "…" } } + ``` + + Unions into `declaredSecrets`. Validate it in `nopy.config.ts` alongside the + other properties. + +**Verify.** New test in `tests/common.test.ts`: `env` carrying a key that cube A +declares secret and cube B does not → B's `get()` does not contain the key; A's +does and is redacted. New test in `tests/executor.test.ts`: the printed plan for +B contains no occurrence of the value. + +### 1.2 `env` must satisfy the `--use-defaults` gap check + +**What happens.** `fillSessionGaps` (`cubes/dependencies.ts:79-89`) builds +`gaps` as `missingRequired ∪ cube.secrets` — *unconditionally* including every +secret, regardless of whether anything supplied a value. Under `-D` it throws, +and the message tells you to do the thing you have already done: + +``` +Error: Cube "user:add" cannot be replayed with --use-defaults: PASSWORD would +have to be entered. … set the values under "env" in .nopyrc.json. +``` + +The value **was** read — without `-D` the prompt came pre-filled from `env`. + +**Fix.** Under `useDefaults`, a gap is satisfied when something outside the +session supplied it deliberately: + +```ts +const unsatisfied = gaps.filter((key) => { + const origin = this.variables.of(cube.id, key)?.origin; + return origin !== 'env' && origin !== 'param'; +}); +if (unsatisfied.length > 0) throw new Error(…); +``` + +`default` is deliberately **not** accepted for a secret. On a replay the +recorded value is gone by design, so falling through to a manifest default would +deploy a different credential than the run being replayed — silently. The +message says so, instead of repeating advice that already failed: + +> `Cube "user:add" cannot be replayed with --use-defaults: PASSWORD is a secret +> and secrets are never recorded in a session. Set it under "env" in +> .nopyrc.json (a schema .default() is not accepted for a secret), pass it from +> a dependency, or replay without --use-defaults.` + +**Verify.** `tests/cubes.dependencies.test.ts`: `-D` replay with the secret under +`env` succeeds and the value reaches the deploy call; with only a schema +`.default()` it throws and the message names the key. Both are new cases. + +### 1.3 Documentation + +`README.md:291` currently prescribes exactly the leak. After 1.1 and 1.2 the +advice becomes true; add one sentence under *Secrets* stating the new rule — a +declared secret in `env` reaches only the cubes that declare it — so the +interaction between the two features is written down once, in the place a reader +of either lands. + +--- + +## 2. Remove `--json` + +Audit §1.2, independently confirmed: `nopy install -R --json > j.out` → +`json.load()` raises; stdout carries seven ANSI-coloured log lines and no JSON. +**Closed by deletion, not by implementation.** + +**Why removal is the right call and not just the cheap one.** `executeDeployCalls` +runs pyinfra through execa with *inherited* stdio (`nopy.executor.ts:117`), so +during a real run nopy does not own its own stdout — pyinfra does, and writes an +unbounded amount to it. A JSON blob appended after that is not machine-readable +by any definition a caller could rely on; making it so means capturing pyinfra's +output and giving up live progress, which is a real feature traded for a +speculative one. That is the same root cause as the documented gap that +`ExecutionResult.stdout` is never populated. The CI case the flag was for is +already covered: `--print-only` for the plan and the exit code for the verdict +(`nopy.cli.ts:138-140` exits 1 on any failure). Nothing can depend on the current +behaviour, because there is no current behaviour. + +### 2.1 The flag, from the `install` command + +- `nopy.cli.ts:91` — drop the `.option('-j, --json', …)` line. +- `:133` — drop `jsonOutput: options.json` from the `nopy()` call. +- `:147-160` — the `if (options.json)` error branch collapses to the single + `console.error`. This is the same statement [§4.3](#43-routine-errors-print-a-raw-node-stack-trace) + rewrites, so whichever phase lands first does both; the other just reads it. + +### 2.2 `jsonOutput`, from the library + +- `NopyOptions.jsonOutput` (`nopy.main.ts:110`) and its destructure (`:141`). + This is a **breaking change to an exported interface** — `docs/API.md:273` + documents it. It is a `0.x` minor bump, and an unknown property is a type error + rather than a silent behaviour change, so a consumer finds out at compile time. +- `:148` — the banner guard becomes `if (!replaySession && !loadSessionPath)`. +- `:158` — the JSON error dump goes; `log.error` on `:156-157` already reported + the same errors. +- `:227` — the `onProgress` callback loses its guard and always logs. +- `outputExecutionPlan(calls, asJson?)` (`nopy.executor.ts:148-158`) — drop the + parameter and the dead JSON branch. Exported and documented (`docs/API.md:588`); + nothing in `src/` passes the second argument, only a test does. + +### 2.3 stdout hygiene — the one fix that survives, and now matters more + +With `--json` gone, `--print-only` is the machine-readable surface, so it has to +be clean. Two writers currently pollute it, and `jsonOutput` was the only thing +holding either back: + +- `configureLogtape`'s console sink uses `console.log` (`nopy.main.ts:34`), + against `README.md:409`, which promises stderr. Switch to `console.error`. +- `printActiveConfig` ends in `console.log` (`nopy.main.ts:95`) and is suppressed + today only by `jsonOutput` and by replay. Same switch. + +Write the rule down once, in the README: **stdout carries the deploy commands and +pyinfra's own output; everything nopy says about itself goes to stderr.** That is +a promise a test can hold to, which the old `--json` claim never was. + +Ripple worth knowing before starting: `tests/main.test.ts` spies on `console.log` +(`logSpy`) throughout, so moving logtape to `console.error` means moving those +spies. Mechanical, but it touches most of the file. + +### 2.4 Documentation — most of the work + +| File | Change | +| --- | --- | +| `packages/nopy/README.md:537-544` | Delete the *JSON output (for CI/CD)* block. Replace with the CI recipe that works: `--print-only` for the plan, exit code `1` for the verdict, `--continue-on-error` when you want every failure in one run. | +| `packages/nopy/README.md:409` | The stderr promise stays and is now load-bearing; reword its reason from `--json` to `--print-only` and piped stdout. | +| `docs/API.md:273` | Remove the `jsonOutput` row from the `NopyOptions` table. | +| `docs/API.md:588-596` | `outputExecutionPlan(calls, asJson?)` → `outputExecutionPlan(calls)`; the note that `--dry-run --json` prints the text plan goes with it. | +| `docs/API.md:1034` | Reword the stderr note the same way as `README.md:409`. | +| `docs/API.md:1170-1173` | *Known gaps*: the `--json` entry disappears — that is the point. `ExecutionResult.stdout` is never populated **stays**, and gains the reason (stdio is inherited), since that is now the honest answer to "how do I capture output?". | +| `docs/CUBE-PACKAGES.md:317` | Future-work line proposes surfacing a cube's source "in the interactive picker and in `--json` output"; drop the second half. | +| `nopy.exit.ts:77` | Comment cites `--json` and `--print-only` as the reason for the exit discipline; leave the discipline, drop the `--json` half. | +| `DOCS-AUDIT.md:85` | Mark §1.2 closed **by removal** and say so in one line — a reader of that document should not go looking for the fix. Also touch its back-references at `:446` and `:854`. | + +### 2.5 `nopy history --json` is a different flag — keep it + +`nopy.cli.ts:169-177` is a second, unrelated `-j, --json`, on the `history` +command, and it works: `JSON.stringify(listHistory())`. It was never part of +audit §1.2 — the field run used it successfully. Keep it. Nothing else writes to +stdout during `history`, so it has none of the problem above, it is three lines, +and it is how a script finds the id to pass to `-H`. + +If the intent is that nopy has no JSON surface at all, removing it is +`nopy.cli.ts:169` plus `:173-177`, and `README.md:541` and `:573`. Flagging it +rather than deciding it: this one is a working feature, so deleting it is a +different kind of change from deleting one that never worked. + +### 2.6 Tests + +Delete, rather than adapt — they assert behaviour that no longer exists: + +- `tests/main.test.ts:162-168` — *emits the errors as JSON when jsonOutput is set* +- `tests/main.test.ts:224-227` — *is suppressed for JSON output* (the sibling + `replaySession` / `loadSession` suppression cases stay and still cover `:148`) +- `tests/main.test.ts:366-373` — *stays silent on progress when jsonOutput is set* +- `tests/executor.test.ts:129` — the `outputExecutionPlan(calls, true)` case + +Add one that holds the new rule: run with `printOnly` and assert `console.log` +received the command block and **nothing else** — banner and progress lines on +`console.error`. Deleting a covered branch moves coverage up, not down, so the +gate is not at risk here. + +--- + +## 3. Replay and session correctness + +### 3.1 `--save-session` no-ops on a replay (audit §4.5) + +`nopy.main.ts:199` guards with `!workflow.isReplay`, so +`nopy install -R -s out.json` exits 0 and writes nothing. Drop the guard: the +resolved cube set is exactly what the user asked to capture, and a replay's +session is no less valid than a fresh run's. + +### 3.2 A `--load-session` replay is not recorded + +`nopy.main.ts:203` excludes every replay from history. For `-R` and `-H` that is +right and documented (`README.md:494`) — repeating must not push the original +out of the list. For `-l` it is wrong: the run is not already in history, so +after deploying from a session file `nopy history` says *"No sessions in +history"* and `-R` has nothing to repeat. That is what happened in the field run. + +Record `-l` runs; keep `-R`/`-H` non-recording. `WorkflowResult` needs to +distinguish them — replace the boolean `isReplay` with +`replaySource: 'file' | 'history' | undefined`, or add a second flag. Then fix +`README.md:496-500`, whose explicit *"a run is not recorded when"* list omits +replays entirely and so contradicts `:494`. + +### 3.3 The written session does not match the documented format (audit §2.4) + +Documented (`README.md:224-254`, `docs/SESSION_FORMAT.md`) versus written: + +| Field | Documented | Written | +| --- | --- | --- | +| `version` | `"1.0.0"` | absent | +| `name` | `"My Deployment Session"` | absent | +| `timestamp` | ISO 8601 | absent | +| `auth.method` | `"ssh-key"` | `"ssh"` | +| `auth.username` | `"root"` | absent | + +This is what you consult in order to hand-write a session, which is what the +field run had to do. + +Implement rather than delete — all three fields are cheap and two are useful: + +- `createSession` (`nopy.session.ts:183-197`) stamps `version: '1.0.0'` and + `timestamp: new Date().toISOString()`, and derives a default `name` the way + `generateEntryName` already does for history (`nopy.history.ts:84-102`). +- `loadSession` (`:130-158`) keeps accepting sessions without them — every + existing file and every hand-written one must stay loadable. Warn on a + `version` it does not know; do not fail. +- `auth.method: 'ssh'` is real, not a bug: `runInteractiveWorkflow:64-67` uses it + for `@vagrant/` and `@docker/` hosts, where the connector owns authentication. + It is simply undocumented. Document the third value and when it appears. + +### 3.4 `listSessions` does not match the documented filename (audit §2.5) + +Docs say `.nopysession.json`; `listSessions` (`nopy.session.ts:166-175`) filters +for `.session.json` / `.session.mjs`, which `wild.nopysession.json` does not +match. Widen the filter to `.nopysession.json` / `.nopysession.mjs` and keep the +old suffixes. + +--- + +## 4. The first five minutes + +The four roughest edges a new user meets all sit before anything that works +well. + +### 4.1 The documented install command 404s + +`packages/nopy-cubes-core/README.md:9`, `packages/nopy/README.md:317` and `:344` +all open with: + +```sh +pnpm add -D @bitsquare/nopy-cubes-core +``` + +``` +[ERR_PNPM_FETCH_404] GET https://registry.npmjs.org/@bitsquare%2Fnopy-cubes-core: Not Found +``` + +The bundle has never been published to npmjs, and an *untagged* Gitea install +resolves to nothing because Gitea publishes no `latest` tag. What rescued the +field run was pnpm's own error listing `main: 0.5.0-main.17.gda84523`. + +Replace both snippets with the form that works, and say why: + +```sh +pnpm add -D @bitsquare/nopy-cubes-core@main \ + --@bitsquare:registry=https://gitea.bitsquare.dev/api/packages/BitSquare/npm/ +``` + +`nopy-cubes-core`'s README does not mention Gitea at all; nopy's mentions it only +in a *Channels* section framed around installing the CLI. Both need the tag +requirement stated where the install command is, not two sections away. Revisit +when `release.yml` first ships the bundle to npmjs — the guard in that workflow +blocks the first `nopy` release until it does. + +### 4.2 pyinfra is an unstated prerequisite + +Nothing in the README says pyinfra must be installed separately and on `PATH`; +`nopy.executor.ts:117` spawns it directly. The field run only worked because it +happened to be there. Add a *Requirements* block next to the install command: +Node ≥ 22, `pyinfra` on `PATH` (`pipx install pyinfra`), plus whatever the chosen +connector needs (`vagrant`, `docker`). Optionally probe for it once at startup +and fail with one line instead of a spawn error. + +### 4.3 Routine errors print a raw Node stack trace + +`nopy.cli.ts:159` passes the error object as a third argument: + +```ts +console.error('Error:', error instanceof Error ? error.message : error, error); +``` + +so the message prints, then the whole error prints again with frames into +`dist/`. Running outside a project — the most likely first-run mistake — yields: + +``` +Error: No .nopyrc.json found. Create one in your project directory or any parent directory. + at loadConfig (…/dist/nopy.config.js:187:15) + at Command. (…/dist/nopy.cli.js:74:24) + at process.processTicksAndRejections (node:internal/process/task_queues:105:5) +``` + +Drop the third argument; print the stack only under `NOPY_DEBUG`. Adopt keyman's +shape (`keyman.cli.ts` is the error boundary that turns a `UsageError` into one +line) so the two CLIs stay in step: a `NopyUsageError` for the errors that are +the user's to fix — no config, no cubes, missing required variable, unknown +session — and a stack for everything else. + +### 4.4 Survive a terminal that reports no size + +Per [§0](#0-retractions): with `stdout.rows === 0`, every enquirer array prompt +renders "No matching choices", the form submits `{}`, and nopy deploys with every +variable defaulted. Reachable outside a test harness — some CI pseudo-terminals, +`script -q`, and editor terminals during startup all report 0 rows. + +Passing an explicit `limit` does **not** help (measured): enquirer clamps it with +`Math.min(limit, this.height)`. But `height` itself has an escape hatch one line +above the bug — `prompt.js:396`: + +```js +get height() { return this.options.rows || utils.height(this.stdout, 25); } +``` + +`options.rows` short-circuits the broken function entirely, so the fix is to pass +a floored size rather than to fake a stdout: + +```ts +const MIN_ROWS = 24, MIN_COLS = 80; +const terminalSize = (out = process.stdout) => ({ + rows: Math.max(out.rows || 0, MIN_ROWS), + columns: Math.max(out.columns || 0, MIN_COLS), +}); +``` + +Measured, 2×2: + +| PTY | without | with | +| --- | --- | --- | +| 0×0 | `RESULT {}` | `RESULT {"USER":"X","PASSWORD":"changeme","GROUPS":"","PUBKEY":""}` | +| 50×200 | full result | full result (`rows` passes through as 50) | + +Apply to both enquirer call sites — `CubeSelection` (`nopy.prompts.ts:61-68`) and +`VariableAssignment` (`:238-243`) — and derive `pageSize` (`:55-56`) from the same +helper, where `process.stdout.rows || 24` already fails for `0` only to be clamped +away again. + +*(An earlier draft of this section proposed a `Proxy` over `process.stdout` +reporting the floor. It works — also measured — but it fakes a stream object to +reach a value the prompt will take directly. `options.rows` is the same fix +without the impersonation.)* + +enquirer 2.4.1 is the last release (2023) and this is its bug. Worth a comment at +the call site so nobody "simplifies" the sizes away later. + +### 4.5 Never deploy a cube with a missing required variable + +Field report §3.7. `README.md:99` guarantees *"Every key defined in the manifest +`schema` is guaranteed to be present on `host.data`"*, and the interactive path +does not enforce it: `resolveCube` calls `VariableAssignment` +(`cubes/dependencies.ts:138`) and goes straight to `buildDeployCall`. +`assertVariablesComplete` exists and runs **only** under `useDefaults` (`:136`). +`buildDeployCall` then emits `--data` for whatever variables exist +(`:186-189`), so a key nothing ever assigned is absent from the command +entirely and the deploy script reads `None`. + +Two ways in, both real: a form that submits nothing (§4.4), and a form the user +cancels — `VariableAssignment`'s `catch {}` (`nopy.prompts.ts:252-254`) swallows +cancellation and returns as though it succeeded. + +- Call the completeness check on the interactive path too, with a message that + fits: `Cube "user:add" is missing PUBKEY. It has no default value and nothing + supplied one.` (The replay path already does this at + `cubes/dependencies.ts:94-100`.) +- Distinguish cancel from error in `VariableAssignment` and route a cancel + through `nopy.exit.ts` like the other prompts, instead of continuing with a + half-filled cube. + +**Verify.** `tests/cubes.dependencies.test.ts`: a cube with a required +no-default key, with the form stubbed to return `{}` → resolution throws and +names the key. This test fails today. + +### 4.6 `self-update` prints a command that cannot work + +From a project with no scope mapping in `.npmrc`: + +``` +Channel: main +Registry: https://registry.npmjs.org/ +Available: unknown +Would run: npm install --global @bitsquare/nopy@main +``` + +`main` snapshots exist only on Gitea, and `buildSelfUpdateCommand` +(`nopy.update.ts:378-381`) deliberately omits the registry flag when the registry +*is* npmjs — correct in general, wrong for this combination. The channel is +derived from the running version, so nopy already knows the command is +unrunnable. + +Detect `channel === 'main' && registry === NPMJS_REGISTRY` in the CLI action and +refuse with a line that fixes it: + +> `You are running a main snapshot, which is published to Gitea only, but +> @bitsquare resolves to npmjs. Re-run with --registry , or set it once: +> npm config set @bitsquare:registry ` + +**Verify.** `tests/update.test.ts` already covers `buildSelfUpdateCommand`'s +registry logic; add the combination case. + +--- + +## 5. Cube defects + +### 5.1 `GLOBAL_PACKAGES` is accepted and then ignored + +`runtime/nodevm/deploy.py:9` reads `GLOBAL_PACKAGES` off `host.data` and never +uses it; `:49` hardcodes the list. The field run passed +`GLOBAL_PACKAGES=npm-check-updates`, watched it appear in the plan and on the +command line, and found it absent from `npm ls -g`. + +```python +"npm install -g pm2 yarn local-web-server node-gyp inquirer execa @dotenvx/dotenvx" +``` + +Fix both halves so behaviour does not change for anyone who never set the +variable — use the parameter in the deploy, and make the manifest default the +list that is hardcoded today: + +```python +f"npm install -g {GLOBAL_PACKAGES}" +``` + +```js +GLOBAL_PACKAGES: z.string() + .describe('Space-separated list of global npm packages to install') + .default('pm2 yarn local-web-server node-gyp inquirer execa @dotenvx/dotenvx'), +``` + +The current default (`npm-check-updates`) is not what the cube installs, so +today's default is wrong in both directions. + +### 5.2 `runtime/nodevm/README.md` describes a different cube (audit §5.3) + +| README says | Manifest / `deploy.py` | +| --- | --- | +| "This cube currently has no configurable parameters" (`:44`) | `VERSION`, `USER`, `ALIAS`, `GLOBAL_PACKAGES`, and `SHELL` after [§5.3](#53-runtimenodevm-has-an-undeclared-shell-dependency--add-a-shell-parameter) | +| "official NodeSource setup script" (`:22`) | `nvm` — `deploy.py:34-37` | +| "Installs the latest LTS version" (`:23`) | whatever `VERSION` says, default `v22.20.0` | +| "npm@11.1.0" in the global list (`:33`) | not installed | +| "Node.js is installed system-wide" (`:80`) | per-user under `~/.nvm` for `USER` | + +Rewrite against the manifest. It is the only file that would tell a reader +`VERSION` or `USER` exist. `runtime/docker/README.md` makes the same +"no configurable parameters" claim with a `DISTRO` field — same fix, same commit. + +### 5.3 `runtime:nodevm` has an undeclared shell dependency — add a `SHELL` parameter + +`dependencies: () => []`, but `deploy.py` runs `omf install nvm` (`:35`) and sets +`_shell_executable='/usr/bin/fish'` (`:43`) — it needs fish **and** Oh My Fish +already installed for `USER`. In the field run it worked only because `user:add` +ran first and installs both. Declaring `user:add` as a dependency would be wrong: +it would create a user that is usually meant to already exist. + +**Fix.** Make the shell a parameter — `SHELL: 'fish' | 'bash'` — so the cube can +be standalone, as its manifest already claims, without taking fish away from +anyone using it today. + +```js +SHELL: z + .enum(['fish', 'bash']) + .describe('Login shell to install through. fish needs Oh My Fish; bash needs nothing') + .default('fish'), +``` + +**Default stays `fish`, deliberately.** nvm wires itself into whichever shell +installed it, so switching the default would leave an existing user — whose login +shell `user:add` set to fish — with node installed and invisible. Additive +change; the escape hatch for a fresh host is one variable. + +Three places differ, and only three: + +| | `fish` | `bash` | +| --- | --- | --- | +| `_shell_executable` | `/usr/bin/fish` | `/bin/bash` | +| Loading nvm | `omf install nvm` — the plugin defines `nvm` as a fish function that every login shell loads | `export NVM_DIR="$HOME/.nvm"; . "$NVM_DIR/nvm.sh"` | +| Making `npm` reachable | the plugin activates the `default` alias on load | `nvm use ` first | + +The bash arm has one non-obvious constraint: **every entry in `commands` is its +own shell**, so sourcing `nvm.sh` and using `nvm` have to be a single entry. +Sourcing cannot be skipped either — nvm's installer appends to `~/.bashrc`, and +Ubuntu's `~/.bashrc` returns at line 1 for a non-interactive shell, so the hook +never runs under `su -c`. fish has no equivalent problem, which is presumably why +it was chosen. + +That same constraint makes `set -gx NVM_DIR $HOME/.nvm` (`:38`) **dead today** — +its own shell, exported, exits. Drop it; the fish plugin sets `NVM_DIR` itself. + +**Guard.** With `SHELL: 'fish'` on a host without fish, fail early and legibly +rather than inside `omf`: + +```python +if SHELL == 'fish' and not host.get_fact(Which, 'fish'): + raise DeployError( + f'runtime:nodevm: SHELL is "fish" but fish is not installed for {USER}. ' + 'Run user:add first, or set SHELL=bash.' + ) +``` + +Deliberately checks the binary only. Oh My Fish is a set of fish functions with +no binary to probe and no fixed path, so a check for it would be guesswork; if +fish is present and omf is not, `omf install nvm` fails with its own clear +message. Half a guard that is certain beats a whole one that is not. + +**While in the file** — `deploy.py:1-6` imports `npm` and `python` and never uses +them, and assigns `hasNode = host.get_fact(Which, 'node')`, also unused. The +`Which` import stops being dead the moment the guard lands. + +**Verify.** No test harness reaches a cube deploy script, so this is acceptance, +not unit: [§7.3](#7-harness-fix-and-acceptance-run) runs `runtime:nodevm` with +`SHELL=bash` on a **fresh** VM where `user:add` has not run, and confirms +`node -v` and the `GLOBAL_PACKAGES` list for `USER`. That is the case the cube has +never survived. + +### 5.4 `VERSION` accepts `null` and would install `None` + +`z.nullable(z.string()).default('v22.20.0')`, and `deploy.py:36` interpolates it +straight into `nvm install {VERSION}`. Either drop `nullable`, or handle `None` +as "latest LTS" — which is what the README claims the cube does anyway. + +### 5.5 `user/add/README.md` — trim, do not rewrite + +Every claim it makes was verified true in the field run, and its notes on *why* +`PUBKEY` has no default and *why* the generated password was removed are the best +documentation in the repo. Its last ~50 lines are generic Fish keybinding tips +(`Ctrl+L` → clear the terminal) unrelated to the cube. Move them somewhere they +belong or delete them; leave the rest alone. + +--- + +## 6. Documentation sweep + +Small, mechanical, no code. + +| File | Change | +| --- | --- | +| `docs/VAGRANT.md` | Never states the `@vagrant/` host syntax — the field run inferred it from an unrelated `@docker/` example. Add `"hosts": ["@vagrant/nopytestvm"]` and one line tying the VM name to it. Add `vagrant destroy -f` for cleanup. | +| `README.md` (root) | Lists a `cubes/` directory at the repo root that no longer exists (`:10`); describes `typecheck` as `tsc --build --noEmit` (`:31`), which TS rejects outright for a project with references. | +| `packages/nopy/README.md` | Top-level `--help` lists only `-V`/`-h`, then the *Examples* block uses `-R`, `-n`, `-P`, `-l`, `-s` — all of which live on `install`. Either add a "these are `install` options" line to the help text (`nopy.cli.ts:56-76`) or promote the common ones. | +| `packages/nopy/README.md` | The host picker offers `docker`, `vagrant`, `@vagrant/…`, `custom`; the first two appear in no document. Add the two connector shortcuts and what they prompt for. | +| `docs/SESSION_FORMAT.md` | Uses `.session.json` throughout while the README uses `.nopysession.json`. Pick one — `.nopysession.json` — and align both, together with [§3.4](#34-listsessions-does-not-match-the-documented-filename-audit-25). | + +Also worth stating once, somewhere prominent: the accuracy failures cluster on +one seam. Everything a human reads on screen matched the docs; everything +machine-facing had drifted — `--json`, the session format, `-s` on replay, `-l` +and history, `nodevm`'s parameters, the install command. That is not random rot, +it is the interactive surface being maintained by daily use while the scripting +surface was documented from intent. Phases 2 and 3 are the correction, in the two +ways available: `--json` was documented from intent and never built, so it goes; +the rest was built and then drifted, so it gets fixed. Keeping it corrected means +what remains of the scripting surface — `--print-only`, sessions, history — +needs tests that assert on **stdout**, not prose. + +--- + +## 7. Harness fix and acceptance run + +**7.1** Fix the PTY driver before it lies again: `drive.py` and `expect.py` must +issue `TIOCSWINSZ` after `pty.fork()`. + +```python +fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) +``` + +Worth keeping the drivers — they are the only way to test the TUI end to end — +so they belong in the repo under `scripts/`, not in a temp folder. + +**7.2** Add a 0-rows regression test that exercises §4.4 directly: spawn the CLI +under a 0×0 PTY and assert the form still yields values. It has to be a real +child process for the same reason `cubes.resolve-hook.test.ts` does — inside a +vitest worker there is no TTY to misreport. + +**7.3** Acceptance: re-run the field scenario from an empty directory — +`vagrant up`, install the bundle with the [§4.1](#41-the-documented-install-command-404s) +command, deploy `user:add` and `runtime:nodevm` **interactively** (not from a +hand-written session), then check: + +- `npm ls -g` contains what `GLOBAL_PACKAGES` asked for (§5.1) +- on a **second, fresh** VM where `user:add` has *not* run, `runtime:nodevm` with + `SHELL=bash` installs node and the global packages; with `SHELL=fish` it fails + in one line naming the missing shell rather than inside `omf` (§5.3) +- `nopy install -P 2>/dev/null` prints the deploy commands and nothing else — no + banner, no progress lines, no update hint (§2.3) +- `nopy install --json` is rejected as an unknown option (§2.1) +- `nopy history` lists the `-l` run (§3.2) +- a dry run with a secret under `env` prints `********` on every cube (§1.1) +- `nopy` in an unconfigured directory prints one line (§4.3) + +### What the run found + +Every check above passed. One deviation and three findings. + +**Deviation.** The bundle was installed from `pnpm pack` tarballs of the three +packages rather than from Gitea, because the cube fixes this plan makes are not +in any published snapshot and publishing one means pushing to `main`. The install +still goes through `cubePackages` → `node_modules` → `/cubes`, which is the +part §4.1 is about; what it does *not* exercise is the registry and dist-tag half +of the documented command. + +Two VMs, as specified: the first got `user:add` then `runtime:nodevm` with +`SHELL=fish`, the second (destroyed and recreated, no fish, no `user:add`) got +`runtime:nodevm` alone under both shells. Driven through `scripts/expect.py`, so +the interactive path is what was exercised. + +**Findings**, all recorded in `DOCS-AUDIT.md`: + +- §6.8 — `--print-only` was recorded in history where `--dry-run` was not, so a + `-P` pass displaced the last real deployment at the head of what `-R` repeats. + Fixed. +- §6.9 — picking `user:add` and `runtime:nodevm` together resolves nodevm first. + The first write-up blamed the ordering and was wrong: emission is already + post-order over `dependencies()`, so a declared edge wins over list order + whichever way round the two were listed, and a test now pins that. What list + order decides is where a cube with *no* edge lands — and `runtime:nodevm` + declares none, deliberately, because `user:add` creates a user. §5.3's + `DeployError` is the guard for that pair; the acceptance run used two + invocations. +- §6.10 — `runtime:nodevm` installed apt packages without refreshing the index, + which only surfaced once §5.3 let the cube run on a box where `apt:essentials` + had not. Fixed. + +--- + +## Suggested order + +1. **Phase 1** — the leak. Security, and the fix is contained. +2. **Phase 4.3–4.5** — the error boundary, the terminal proxy, and the + completeness check. Small, and they stop a silently wrong deployment. +3. **Phase 2** — remove `--json`. Mostly deletion, and it settles what the + scripting surface *is* before phase 6 documents it. +4. **Phase 6 + 4.1 + 4.2** — documentation. No code, immediate payoff for the + next new user. +5. **Phase 5** — cubes. Independent of everything above; ships with the bundle, + not the CLI. +6. **Phase 3** — session and replay. Largest surface, lowest severity. +7. **Phase 7** — harness and acceptance, last, so it exercises all of it. diff --git a/README.md b/README.md index 4fdcb3e..303da13 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Infrastructure tooling monorepo: two published CLIs plus the pyinfra "cubes" they deploy. -| Path | Package | Binary | What it is | -| ----------------- | ------------------ | -------- | --------------------------------------------------- | -| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution | -| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management with `age` encryption | -| `cubes/` | — | — | the deployment units `nopy` runs | +| Path | Package | Binary | What it is | +| -------------------------- | ---------------------------- | -------- | --------------------------------------------------- | +| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution | +| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management with `age` encryption | +| `packages/nopy-cubes` | `@bitsquare/nopy-cubes` | — | the authoring surface a cube's `manifest.mjs` imports | +| `packages/nopy-cubes-core` | `@bitsquare/nopy-cubes-core` | — | the core bundle of deployment units `nopy` runs | ```sh npm install -g @bitsquare/nopy @bitsquare/keyman @@ -28,7 +29,7 @@ pnpm install | Command | Does | | --------------------------- | --------------------------------------------------- | | `pnpm run build` | compiles both packages with `tsc` | -| `pnpm run typecheck` | `tsc --build --noEmit` across the workspace | +| `pnpm run typecheck` | `tsc --build` across the workspace (see below) | | `pnpm run lint` | Biome check | | `pnpm run lint:fix` | Biome check with fixes applied | | `pnpm test` | vitest, both packages | @@ -36,7 +37,11 @@ pnpm install | `pnpm run coverage:summary` | renders the last coverage run as a Markdown table | `typescript` is on the 7.x native compiler, so `tsc` *is* the fast one — there is -no separate `tsgo` binary to keep in sync. Each package also has a dev-run script +no separate `tsgo` binary to keep in sync. `typecheck` is plain `tsc --build`, +not `--noEmit`: once a project has `references`, TypeScript rejects `--noEmit` +outright (TS6310), because a composite project has to emit the declarations its +dependents read. So the typecheck writes `dist` as a side effect — gitignored, +and it means the gate also proves the build works. Each package also has a dev-run script (`pnpm --filter @bitsquare/nopy run nopy`) that executes the TypeScript sources directly through `tsx`. diff --git a/packages/nopy-cubes-core/README.md b/packages/nopy-cubes-core/README.md index 0faa951..adbbd66 100644 --- a/packages/nopy-cubes-core/README.md +++ b/packages/nopy-cubes-core/README.md @@ -6,9 +6,18 @@ base packages, users, SSH, firewalling, networking, web serving and runtimes. ## Install ```sh -pnpm add -D @bitsquare/nopy-cubes-core +pnpm add -D @bitsquare/nopy-cubes-core@main \ + --@bitsquare:registry=https://gitea.bitsquare.dev/api/packages/BitSquare/npm/ ``` +**Both halves are required today.** This bundle has not been published to npmjs +yet, so it comes from the Gitea registry — and that registry publishes no +`latest` tag, so an *untagged* install resolves to nothing at all. Name `@main` +(a snapshot of every commit) or `@next` (a prerelease) explicitly. Point the +**scope** at Gitea rather than setting a bare `registry=`: Gitea serves +`@bitsquare` only and does not proxy npmjs, so everything else must keep +resolving from there. Reading needs no token while the repository is public. + Then name it in `.nopyrc.json`: ```json diff --git a/packages/nopy-cubes-core/cubes/runtime/docker/README.md b/packages/nopy-cubes-core/cubes/runtime/docker/README.md index a9fd3f5..b863603 100644 --- a/packages/nopy-cubes-core/cubes/runtime/docker/README.md +++ b/packages/nopy-cubes-core/cubes/runtime/docker/README.md @@ -35,7 +35,9 @@ Key benefits: ## Configuration -This cube currently has no configurable parameters. +| Variable | Default | What it does | +| --- | --- | --- | +| `DISTRO` | `ubuntu` | which of Docker's package repositories to add — `ubuntu` or `debian`. It selects the download path and nothing else; the release codename comes from the host's own `/etc/os-release`. | ## Dependencies diff --git a/packages/nopy-cubes-core/cubes/runtime/nodevm/README.md b/packages/nopy-cubes-core/cubes/runtime/nodevm/README.md index 8f75a87..02e98ac 100644 --- a/packages/nopy-cubes-core/cubes/runtime/nodevm/README.md +++ b/packages/nopy-cubes-core/cubes/runtime/nodevm/README.md @@ -1,69 +1,79 @@ # nodevm -**Install Node.js with essential global packages** +**Install Node.js through nvm, for one user, with global packages** ## Purpose -This cube installs the latest LTS (Long Term Support) version of Node.js along with essential global npm packages commonly needed for development and deployment. +Installs [nvm](https://github.com/nvm-sh/nvm) into a single user's home +directory, uses it to install one pinned Node.js version under an alias, and +installs a list of global npm packages for that user. -## What is Node.js? - -Node.js is a JavaScript runtime built on Chrome's V8 engine that allows you to run JavaScript on the server. It's widely used for: - -- Building web servers and APIs -- Command-line tools -- Build tools and task runners -- Real-time applications (chat, notifications) -- Microservices +Per-user, not system-wide. Nothing is placed on the system `PATH`, and another +user on the same host is unaffected — which is the point: version pinning belongs +to whoever runs the app. ## What This Cube Does -1. **Installs Node.js LTS** - - Downloads and runs the official NodeSource setup script - - Installs the latest LTS version of Node.js - - Includes npm (Node Package Manager) - -2. **Installs build dependencies** - - `libssl-dev` - SSL/TLS libraries - - `libtool` - Library building tools - - `cmake` - Cross-platform build system - - `libpng-dev`, `libjpeg-dev`, `libvips-dev` - Image processing libraries - -3. **Installs global npm packages** - - **npm@11.1.0** - Latest npm version - - **pm2** - Production process manager for Node.js apps - - **yarn** - Alternative package manager - - **local-web-server** - Local development web server - - **node-gyp** - Node.js native addon build tool - - **inquirer** - Interactive command-line prompts - - **execa** - Better child process execution - - **@dotenvx/dotenvx** - Environment variable management +1. **Installs build dependencies** with apt, as root — `build-essential`, + `libssl-dev`, `libtool`, `cmake`, and the cairo/pango/png/jpeg/vips/rsvg/pixman + headers that native addons need. The package index is refreshed first: a box + nobody has updated lists .deb versions the mirror has already dropped. +2. **Installs nvm** for `USER` via the official install script, then + `nvm install ` and `nvm alias `. +3. **Installs `GLOBAL_PACKAGES`** with `npm install -g`, as `USER`. ## Configuration -This cube currently has no configurable parameters. +| Variable | Default | What it does | +| --- | --- | --- | +| `VERSION` | `v22.20.0` | the Node.js version nvm installs. A pin, not "latest LTS" — nvm's own version strings work, so `--lts` or `22` are accepted too. | +| `USER` | `vagrant` | the user nvm is installed **for**. Everything lands in that user's `~/.nvm`. | +| `ALIAS` | `nodelts` | the nvm alias pointing at `VERSION`, so later cubes and scripts can say `nvm use nodelts` without knowing the number. | +| `GLOBAL_PACKAGES` | `pm2 yarn local-web-server node-gyp inquirer execa @dotenvx/dotenvx` | space-separated, passed to one `npm install -g`. Setting it **replaces** the list rather than adding to it. | +| `SHELL` | `fish` | the login shell to install through — `fish` or `bash`. See below. | + +### `SHELL` + +nvm wires itself into whichever shell installed it, so this is not cosmetic. + +- **`fish`** (default) additionally requires **Oh My Fish**, because loading nvm + goes through the `omf install nvm` plugin. `user:add` installs both, which is + the usual way a host arrives here. The cube fails with one line, before + changing anything, if `SHELL=fish` on a host with no fish. +- **`bash`** needs nothing beyond bash. Use it on a host where `user:add` has not + run. + +The default stays `fish` so that an existing user — whose login shell `user:add` +set to fish — keeps getting a Node that their shell can actually see. Switching +would install it invisibly. ## Dependencies -None - this cube can run standalone. +None declared: the cube runs standalone. With `SHELL=fish` it does have a real +prerequisite (fish + Oh My Fish, which `user:add` provides), but `user:add` is +deliberately not a declared dependency — it would *create* a user who is normally +meant to already exist. `SHELL=bash` is the standalone path. ## Post-Installation -Verify installation: +`node` is on `USER`'s `PATH` in a login shell, not in root's and not in a +non-interactive one. To check: + ```bash -node --version -npm --version +su - -c 'node --version && npm --version' ``` -Common commands: -- Run a Node.js app: `node app.js` -- Start with PM2: `pm2 start app.js` -- Install packages: `npm install ` -- Use yarn: `yarn add ` +From a bash script that is not a login shell, load nvm first: -## PM2 - Process Manager +```bash +export NVM_DIR="$HOME/.nvm"; . "$NVM_DIR/nvm.sh" +nvm use nodelts +``` -PM2 is included for production deployments. Common PM2 commands: +## PM2 — process manager + +`pm2` is in the default `GLOBAL_PACKAGES`, so it is installed unless you replaced +the list. ```bash pm2 start app.js # Start application @@ -75,9 +85,11 @@ pm2 startup # Enable PM2 on boot pm2 save # Save current process list ``` +`pm2 startup` prints a `sudo` command to run; it does not enable itself. + ## Notes -- Node.js is installed system-wide -- Global packages are accessible to all users -- npm cache is stored in `~/.npm` -- Use `nvm` if you need multiple Node.js versions +- Node.js is installed **per user**, under `~/.nvm` for `USER`. +- Global packages belong to that user too, not to everyone on the host. +- Run the cube again with a different `VERSION` and `ALIAS` to have several + versions side by side; nvm is built for exactly that. diff --git a/packages/nopy-cubes-core/cubes/runtime/nodevm/deploy.py b/packages/nopy-cubes-core/cubes/runtime/nodevm/deploy.py index 93c513a..dad5cb8 100644 --- a/packages/nopy-cubes-core/cubes/runtime/nodevm/deploy.py +++ b/packages/nopy-cubes-core/cubes/runtime/nodevm/deploy.py @@ -1,15 +1,53 @@ -from pyinfra.operations import server, apt, npm, python +from pyinfra.operations import server, apt from pyinfra import host -from pyinfra.facts.files import Directory from pyinfra.facts.server import Which +from pyinfra.api.exceptions import DeployError -hasNode = host.get_fact(Which, 'node') VERSION = host.data.VERSION ALIAS = host.data.ALIAS GLOBAL_PACKAGES = host.data.GLOBAL_PACKAGES USER = host.data.USER +SHELL = host.data.SHELL -apt.packages( +INSTALL_NVM = "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash" + +# Under bash, every entry in `commands` is its own shell, so loading nvm and +# using it have to be one entry. Loading cannot be skipped either: nvm's +# installer appends its hook to ~/.bashrc, and Ubuntu's ~/.bashrc returns at +# line 1 for a non-interactive shell, so under `su -c` the hook never runs. +LOAD_NVM = 'export NVM_DIR="$HOME/.nvm"; . "$NVM_DIR/nvm.sh"' + +# The binary only. Oh My Fish is a set of fish functions with no binary and no +# fixed path, so probing for it would be guesswork — and if fish is there while +# omf is not, `omf install nvm` says so itself. Half a guard that is certain +# beats a whole one that is not. +if SHELL == 'fish' and not host.get_fact(Which, 'fish'): + raise DeployError( + f'runtime:nodevm: SHELL is "fish" but fish is not installed for {USER}. ' + 'Run user:add first, or set SHELL=bash.' + ) + +if SHELL == 'fish': + shell_executable = '/usr/bin/fish' + # The omf plugin defines `nvm` as a fish function that every login shell + # loads, and activates the `default` alias on load — so `nvm` and `npm` are + # both reachable in any later shell without setup, and NVM_DIR is set for us. + nvm_commands = [ + INSTALL_NVM, + 'omf install nvm', + f'nvm install {VERSION}', + f'nvm alias {ALIAS} {VERSION}', + ] + npm_commands = [f'npm install -g {GLOBAL_PACKAGES}'] +else: + shell_executable = '/bin/bash' + nvm_commands = [ + INSTALL_NVM, + f'{LOAD_NVM}; nvm install {VERSION}; nvm alias {ALIAS} {VERSION}', + ] + npm_commands = [f'{LOAD_NVM}; nvm use {ALIAS}; npm install -g {GLOBAL_PACKAGES}'] + +apt.packages( name=f'Install nodejs tools', no_recommends=True, packages=[ @@ -26,33 +64,30 @@ apt.packages( 'librsvg2-dev', 'libpixman-1-dev', ], + # Every other cube that installs packages refreshes the index first, and + # this one only got away without it while `user:add` ran ahead of it and + # dragged in `apt:essentials`. On a box nobody has updated, the index + # names .deb versions the mirror has already superseded and the fetch + # 404s — the same "assumes a predecessor cube ran" defect as the shell. + update=True, _sudo = True, ) server.shell( - commands=[ - "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash", - "omf install nvm", - f"nvm install {VERSION}", - f"nvm alias {ALIAS} {VERSION}", - "set -gx NVM_DIR $HOME/.nvm", - ], + name=f'Install nvm and node {VERSION} for {USER}', + commands=nvm_commands, _sudo=True, _su_user=USER, _use_su_login=True, - _shell_executable='/usr/bin/fish' + _shell_executable=shell_executable, ) server.shell( - commands=[ - "npm install -g pm2 yarn local-web-server node-gyp inquirer execa @dotenvx/dotenvx" - ], + name=f'Install global packages for {USER}', + commands=npm_commands, _sudo=True, _su_user=USER, _use_su_login=True, - _shell_executable='/usr/bin/fish' - + _shell_executable=shell_executable, ) - - diff --git a/packages/nopy-cubes-core/cubes/runtime/nodevm/manifest.mjs b/packages/nopy-cubes-core/cubes/runtime/nodevm/manifest.mjs index e4300ed..dbf265c 100644 --- a/packages/nopy-cubes-core/cubes/runtime/nodevm/manifest.mjs +++ b/packages/nopy-cubes-core/cubes/runtime/nodevm/manifest.mjs @@ -7,14 +7,24 @@ export default Manifest({ dependencies: () => [], schema: z.object({ VERSION: z - .nullable(z.string()) + .string() .describe('Node.js version to install. It is recommended to use semver notation') .default('v22.20.0'), USER: z.string().describe('Username for which to install nodejs').default('vagrant'), ALIAS: z.string().describe('The alias for this node version').default('nodelts'), + // The list the deploy script used to hardcode. It is the default rather than + // a constant so that setting the variable adds to nothing and replaces + // everything — which is what "space-separated list" reads as. GLOBAL_PACKAGES: z .string() .describe('Space-separated list of global npm packages to install') - .default('npm-check-updates'), + .default('pm2 yarn local-web-server node-gyp inquirer execa @dotenvx/dotenvx'), + // fish is the default because nvm wires itself into whichever shell installed + // it: switching would leave an existing user — whose login shell `user:add` + // set to fish — with node installed and invisible. + SHELL: z + .enum(['fish', 'bash']) + .describe('Login shell to install through. fish needs Oh My Fish; bash needs nothing') + .default('fish'), }), }); diff --git a/packages/nopy-cubes-core/cubes/user/add/README.md b/packages/nopy-cubes-core/cubes/user/add/README.md index e4c953c..f07d5c3 100644 --- a/packages/nopy-cubes-core/cubes/user/add/README.md +++ b/packages/nopy-cubes-core/cubes/user/add/README.md @@ -92,52 +92,7 @@ After deployment: - Oh My Fish provides package management: `omf install ` - To switch shells: `chsh -s /bin/bash` (or back to fish: `chsh -s /usr/bin/fish`) ---- - -# 📌 Most Useful Fish Key Bindings (with Fisher Extensions) - -## 🐟 Default Fish Key Bindings - -- `Ctrl + C` → Cancel the current command -- `Ctrl + D` → Exit the shell (or logout if in SSH) -- `Ctrl + L` → Clear the terminal -- `Ctrl + R` → Search command history (enhanced by `fzf.fish`) -- `Ctrl + U` → Delete the entire command line -- `Ctrl + W` → Delete the last word -- `Alt + ← / →` → Move backward/forward by a word - -## 🔍 Enhanced with `fzf.fish` - -- `Ctrl + R` → **Fuzzy search command history** -- `Ctrl + T` → **Fuzzy search and insert file path** -- `Alt + C` → **Fuzzy search directories (`cd` with `z`)** - -## 📂 Directory Navigation (with `z`) - -- `z ` → Jump to a frequently used directory -- `z -l` → List most-used directories -- `z -c` → Remove a directory from `z`'s database - -## 🔄 Process & Job Management - -- `Ctrl + Z` → Suspend the current process -- `fg` → Bring a suspended process back to foreground -- `jobs` → List background jobs - -## 🎨 Other Handy Shortcuts - -- `fish_vi_key_bindings` → Enable Vi mode (press `Esc` for normal mode) -- `Ctrl + G` → Show Git status (if using `fzf.fish`) -- `Ctrl + E` → Edit command line in `$EDITOR` - -## ⚙️ Useful Commands for Key Binding - -```fish -# Set Fish default key bindings -fish_default_key_bindings - -# Enable Vi mode -fish_vi_key_bindings - -# Rebind a custom key (Example: Ctrl + G for git status) -bind \cg 'git status' +Fish's own key bindings and the plugins this cube installs are documented +upstream — `fish_key_reader` lists what is bound, and `omf help` what is +installed. They used to be reproduced here at length, which is not something +this cube knows anything about. diff --git a/packages/nopy-cubes/src/types.ts b/packages/nopy-cubes/src/types.ts index 0355182..1a43577 100644 --- a/packages/nopy-cubes/src/types.ts +++ b/packages/nopy-cubes/src/types.ts @@ -192,6 +192,18 @@ export class Cube { return defaults as z.infer; } + /** + * Every key the schema declares, required or not. + * + * The question this answers is "does this cube claim to know about KEY", which + * is not the same as "does it have a value for it" — a cube can read a key off + * `host.data` that only the config `env` supplies. That distinction is what + * decides whether a secret is allowed to travel to it. + */ + schemaKeys(): string[] { + return Object.keys(this.manifest.schema.shape); + } + /** * Schema keys that have to be supplied from somewhere: no `.default()`, and * not optional. Nothing else can fill them in, so a run that cannot prompt diff --git a/packages/nopy/README.md b/packages/nopy/README.md index 2c610b8..93d2ae0 100644 --- a/packages/nopy/README.md +++ b/packages/nopy/README.md @@ -15,7 +15,7 @@ Nopy wraps pyinfra with structure, validation, and an interactive experience for - **Schema validation** using Zod - **Recursive cube directory discovery** - **Dry-run mode** for previewing deployments -- **JSON output** for CI/CD integration +- **Pipeable output** for CI/CD integration — the plan on stdout, everything else on stderr - **Session history** with replay capability ## Workflow @@ -142,13 +142,25 @@ export default cubes.Manifest({ Every entry must be a key of `schema`; naming anything else is a manifest error and aborts the run, so a typo fails loudly instead of silently leaving a value unprotected. -Declaring a key a secret changes three things: +Declaring a key a secret changes four things: - **It is never written to a session file or to the history.** Everything else the run settled on is recorded — including values that came from a `.default()` — but declared secrets are left out. - **It is masked wherever a command or a plan is printed** — `--dry-run`, `--print-only`, and the debug log all show `********` in place of the value, in the variable list *and* in the `pyinfra` command line above it. The SSH password passed via `--password` is masked the same way, whether or not any cube declares secrets. - **It is re-prompted on replay**, since there is nothing recorded to replay from (see [Session Recording and Replay](#session-recording-and-replay)). +- **It stops travelling.** Ordinary `env` values are seeded onto every cube in the run, because a cube may read a key off `host.data` that its own schema never declared. A secret is the exception: it reaches only the cubes whose `schema` names it. Otherwise putting a password under `env` — which is what unattended replay asks you to do — would put it on the command line of every unrelated cube, where nothing masks it because that cube never called it a secret. -Nopy does not guess. A key called `PASSWORD` in a manifest that declares no `secrets` is treated as an ordinary variable — recorded, and printed in the clear. +Declaring is global, masking is global. A key any manifest calls a secret is masked and kept out of sessions on every cube it lands on, even one whose own manifest forgot to list it. What is *not* global is the guess: a key called `PASSWORD` that no manifest declares anywhere is an ordinary variable — broadcast, recorded, and printed in the clear. + +For a sensitive `env` value that no cube declares at all — a token only a hook reads, say — name it in the config instead: + +```json +{ + "secrets": ["DEPLOY_TOKEN"], + "env": { "DEPLOY_TOKEN": "..." } +} +``` + +Entries here behave exactly like a manifest's: masked, never recorded, and delivered only to cubes that declare them. Three limits are worth knowing, because `secrets` keeps a value out of the files nopy writes and nothing more: @@ -168,6 +180,7 @@ Uses `.nopyrc.json` files (project-level or home directory) containing: "env": { "SHARED_VAR": "value" }, + "secrets": ["DEPLOY_TOKEN"], "log": { "verbosity": "info", "debug": false @@ -184,8 +197,36 @@ Uses `.nopyrc.json` files (project-level or home directory) containing: `history` controls automatic session recording (see [Deployment History](#deployment-history)), and `execution.continueOnError` sets the default for `--continue-on-error`. +`secrets` names `env` keys to treat as sensitive that no manifest declares — it is the config-side half of a manifest's `secrets`, and behaves identically. See [Secrets](#secrets). + +`hosts` seeds the target picker; see [Target hosts](#target-hosts) for what else that picker offers. + `cubeDirs` holds paths, `cubePackages` holds installed npm packages that ship cubes — see [Cube Discovery](#cube-discovery) below and [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md) for publishing your own. Both are additive, and both resolve relative to the config file that named them, not to the working directory: a `.nopyrc.json` two levels up may name a package that only exists in *its* `node_modules`. +#### Target hosts + +The host prompt offers more than the `hosts` array. Two entries at the top are +shortcuts for pyinfra's local connectors, each asking one follow-up question and +assembling the host string from the answer: + +| Picked | Asks for | Becomes | +| --- | --- | --- | +| `docker` | a container name/id, **or** an image reference | `@docker/` | +| `vagrant` | the machine name (default `default`) | `@vagrant/` | +| *(a configured host)* | — | itself | +| `custom` | any address | itself | + +The two connector forms can equally be written into `hosts` directly — a session +records whatever string the run used, so `"hosts": ["@vagrant/nopytestvm"]` and +picking `vagrant` are the same thing to everything downstream. + +The docker answer is deliberately not validated as one kind or the other, because +the two mean very different things and only the connector can tell them apart (it +looks for a matching container first). A **container** is mutated in place and +left running; an **image** makes pyinfra start a throwaway container, apply the +deploy, commit the result as a new image and print its id. See +[DOCKER.md](docs/DOCKER.md) and [VAGRANT.md](docs/VAGRANT.md). + #### Logging Configuration Control pyinfra output verbosity and debug information using the `log` configuration object: @@ -259,12 +300,17 @@ Sessions are stored in `.nopysession.json` files with the following structure: - **`env`**: The `env` block of `.nopyrc.json` as it stood at record time, kept for reference - **`hosts`**: Array of target hosts - **`auth`**: Authentication configuration (passwords are never stored) +- **`version`**, **`timestamp`**, **`name`**: stamped on every session nopy writes — the format version, the ISO 8601 record time, and a one-line description in the same `date - cubes → hosts` form the history list uses + +Only `cubes` and `auth` are required. A hand-written session may omit the rest, and one that predates the stamp still loads; a `version` this build does not recognise is a warning on stderr, never a refusal. + +`auth.method` has a third value the picker never offers: **`ssh`**, meaning the connector owns authentication and nopy supplies none. It is what an `@vagrant/` or `@docker/` host gets, which is why replaying one asks for nothing. **What is recorded:** every value each cube settled on, regardless of where it came from — a value the user typed, one inherited from `.nopyrc.json` `env`, one a dependency supplied, and one that fell through to the schema's `.default()` are all written out the same way. A session is therefore a full snapshot rather than a diff, and a `--use-defaults` run produces a session with real values in it instead of an empty one. The consequence is that replay is faithful rather than re-derived: the recorded value outranks the current `.nopyrc.json` `env` and the current schema default, so editing either one does not silently change what a replay does. To pick up a new default, record a fresh session. -**Security Note**: Passwords are never stored in session files. This covers both the SSH password — a session records the auth *method* and username, never the credential — and any schema key a cube's manifest lists under [`secrets`](#secrets). Both are re-prompted on replay. +**Security Note**: Passwords are never stored in session files. This covers both the SSH password — a session records the auth *method* and username, never the credential — and any schema key a cube's manifest lists under [`secrets`](#secrets). Both are re-prompted on replay. The rule applies to the session's `env` block as well as to each cube's `variables`, so a declared secret set in `.nopyrc.json` is left out of the recorded copy rather than written back out in plaintext. #### Recording a Session @@ -274,6 +320,9 @@ nopy install --save-session my-deployment.nopysession.json # With defaults (no prompts for variables) nopy install -D --save-session automated-deployment.nopysession.json + +# Also works on a replay — the resolved cube set is what you asked to capture +nopy install -R --save-session repeat-of-the-last-run.nopysession.json ``` #### Replaying a Session @@ -288,7 +337,9 @@ nopy install --load-session my-deployment.nopysession.json A replay runs straight through without asking anything, with three exceptions. Password authentication always re-prompts. A session with no recorded host falls back to the host picker. And a cube is re-prompted for its declared secrets, plus for any required variable the session has no value for — which happens when the cube's schema has gained a field since the session was written. -Those re-prompts are what a session cannot supply, so `--use-defaults` cannot paper over them: combining `-D` with a replay that needs either fails with a message naming the keys rather than deploying with a placeholder. Put the values under `env` in `.nopyrc.json` to make such a replay unattended. +Those re-prompts are what a session cannot supply, so `--use-defaults` cannot paper over them: combining `-D` with a replay that needs either fails with a message naming the keys rather than deploying with a placeholder. Put the values under `env` in `.nopyrc.json` — or pass them from a dependency — to make such a replay unattended. A secret supplied that way reaches only the cubes that declare it, so this does not broadcast it across the run; see [Secrets](#secrets). + +A schema `.default()` is deliberately *not* accepted in its place. The recorded answer is gone on purpose, so falling back to the manifest would deploy a different credential than the run being replayed, and say nothing about it. ### Cube Discovery @@ -314,13 +365,17 @@ A cube package is an ordinary npm package that ships its cubes in a `cubes/` dir Install it and name it — nothing needs linking or copying: ```sh -pnpm add -D @bitsquare/nopy-cubes-core +pnpm add -D @bitsquare/nopy-cubes-core@main \ + --@bitsquare:registry=https://gitea.bitsquare.dev/api/packages/BitSquare/npm/ ``` ```json { "cubePackages": ["@bitsquare/nopy-cubes-core"] } ``` +The tag and the registry flag are both required for this bundle today — see +[Installation](#installation). + Naming a package is a statement that cubes are expected from it, so anything wrong is an error that aborts the run rather than a silent skip: the package is not installed, it has neither a `cubes/` directory nor a `nopy.cubes` override, its `nopy.cubes` is malformed, or an entry points at a directory that does not exist or lies outside the package. #### Ids are claimed globally @@ -331,6 +386,18 @@ Writing cubes to publish is covered in [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md). ## Command Line Usage +### Requirements + +| | | +| --- | --- | +| **Node** | ≥ 22 | +| **pyinfra** | on `PATH` — `pipx install pyinfra` | +| **the connector** | `vagrant` or `docker` on `PATH`, if you deploy to one | + +nopy builds pyinfra command lines and spawns them; it does not vendor pyinfra and +will not install it for you. A missing `pyinfra` surfaces as a spawn failure on +the first deploy, after every prompt has been answered. + ### Installation ```bash @@ -341,13 +408,21 @@ The cubes live in a separate bundle, installed into whichever project describes your infrastructure and named in its `.nopyrc.json`: ```bash -pnpm add -D @bitsquare/nopy-cubes-core +pnpm add -D @bitsquare/nopy-cubes-core@main \ + --@bitsquare:registry=https://gitea.bitsquare.dev/api/packages/BitSquare/npm/ ``` ```json { "hosts": ["your-host"], "cubePackages": ["@bitsquare/nopy-cubes-core"] } ``` +**The tag and the registry flag are both required for the bundle today.** It has +not been published to npmjs yet, and the Gitea registry publishes no `latest` +tag, so a plain `pnpm add -D @bitsquare/nopy-cubes-core` fails with a 404 against +npmjs and an untagged Gitea install resolves to nothing. Name `@main` or `@next` +explicitly. See [Channels](#channels) for what the tags mean and how to set the +scope persistently. The CLI itself is on npmjs and installs without either. + #### Channels Three dist-tags are published, and the one you install from is the one you stay @@ -406,8 +481,8 @@ npm install -g @bitsquare/nopy@latest ``` Once a day, `nopy` checks its channel in the background and prints a one-line -hint to **stderr** when a newer version exists — never to stdout, so `--json` -and `--print-only` output stay clean. The answer is cached in +hint to **stderr** when a newer version exists — never to stdout, so a piped +`--print-only` stays clean. The answer is cached in `~/.nopy/update-check.json`; a registry that is slow or unreachable is given 1.5 seconds and then ignored. @@ -493,11 +568,14 @@ Every deployment is automatically recorded to a `.nopy.history.json` file in the The recording happens before the deploy commands run, so a **failed** deployment is recorded too — `-R` is the quick way to retry one after fixing the cause. Replaying a session with `-R` or `-H` does not itself create a new entry, so repeating never pushes the original run out of the list. +A `--load-session` run *is* recorded, and the distinction is the point: a session file has never been in history, so without the entry `nopy history` would report nothing afterwards and `-R` would have nothing to repeat. + A run is *not* recorded when: -- `--dry-run` or `--no-history` is passed +- `--dry-run`, `--print-only` or `--no-history` is passed — the first two deploy nothing, and history is what `-R` repeats - No cubes were selected, so there was nothing to deploy - `history.autoSave` is set to `false` in `.nopyrc.json` +- it is a `-R` or `-H` replay, as above Because the history file is resolved against the current working directory, each project keeps its own history — running nopy from a different directory will not find the previous run. As with session files, passwords are never stored and are re-prompted on replay. @@ -534,14 +612,26 @@ nopy install --dry-run Shows the execution plan including commands, environment variables, and targets without running anything. Sensitive data is masked in output. -**JSON output (for CI/CD)**: +**CI/CD**: ```bash -nopy install --json -nopy history --json +nopy install --print-only > plan.txt # the commands, and nothing else +nopy install -D # run it; exit code 1 if any cube failed ``` -Machine-readable JSON output for scripting and CI/CD integration. +There is no `--json` on `install`, deliberately. A deploy runs pyinfra with +inherited stdio, so during a run nopy does not own its own stdout — pyinfra does, +and writes an unbounded amount to it. Anything nopy appended afterwards would not +be parseable by any definition a caller could rely on. Two things are guaranteed +instead: + +- **stdout carries the deploy commands and pyinfra's own output. Everything nopy + says about itself — the config banner, progress lines, warnings, the update + hint, errors — goes to stderr.** So `--print-only` redirects cleanly. +- **The exit code is the verdict**: `1` if any cube failed, `0` otherwise. + +`nopy history --json` is unaffected and is how a script finds the id to pass to +`-H`. **Continue on error**: diff --git a/packages/nopy/docs/API.md b/packages/nopy/docs/API.md index 5c81dfb..a6ff95f 100644 --- a/packages/nopy/docs/API.md +++ b/packages/nopy/docs/API.md @@ -137,6 +137,7 @@ class Cube { get secrets(): string[]; // manifest.secrets ?? [] getDefaults(): z.infer; + schemaKeys(): string[]; requiredKeys(): string[]; isSecret(key: string): boolean; } @@ -151,6 +152,11 @@ single required field used to leave the cube with no variables at all. and not optional. A `--use-defaults` run that cannot supply one aborts by name rather than deploying the cube with the value missing. +`schemaKeys()` returns every declared key, required or not. It answers a +different question — whether the cube *claims to know about* a key, rather than +whether it has a value for one — and that is what decides whether a secret in the +config `env` is allowed to reach it. + ### `CubeSource` Where a cube came from. Carried because a cube's directory does not say how it @@ -264,13 +270,12 @@ const result = await nopy({ useDefaults: true, dryRun: true }); |------|------|---------|-------------| | `useDefaults` | `boolean` | `false` | Skip the variable prompts. A cube with a required key nothing supplied aborts the run by name. | | `useAuthKey` | `boolean` | `false` | Force SSH key auth, skipping the auth prompt. | -| `saveSession` | `string` | – | Path to write the session to. **Ignored during a replay.** | +| `saveSession` | `string` | – | Path to write the session to. Honoured on a replay too. | | `loadSession` | `string` | – | Path to a session file to replay. | | `replaySession` | `NopySession` | – | A session object to replay, used by `-R` / `-H` from history. Takes precedence over `loadSession`. | | `dryRun` | `boolean` | `false` | Print the execution plan instead of running it. | | `printOnly` | `boolean` | `false` | Print the built pyinfra commands and return; the executor is never reached. | | `continueOnError` | `boolean` | `false` | Keep going after a cube fails. | -| `jsonOutput` | `boolean` | `false` | Suppress the config banner and progress lines. See [Known gaps](#known-gaps). | | `saveToHistory` | `boolean` | `true` | Record the session in `.nopy.history.json`. | **Returns:** `Promise` — `undefined` when cube loading @@ -507,9 +512,10 @@ displaced stays visible underneath. The trace is never persisted. ```typescript class Variables { - constructor(env?: TVariables); + constructor(env?: TVariables, globalSecrets?: Iterable); declareSecrets(cube: string, keys: readonly string[]): void; + declareSchema(cube: string, keys: readonly string[]): void; isSecret(cube: string, name: string): boolean; assign(cube: string, origin: Origin, values?: TVariables): void; @@ -527,6 +533,25 @@ const MASK = '********'; `declareSecrets()` is retroactive as well as prospective, so it does not matter whether the caller declares before or after the values arrive. +`globalSecrets` is every key *any* manifest declares secret, plus the config's +own `secrets` list. `nopy()` computes it once after `loadCubes()`, before the +first cube resolves, so resolution order cannot change whether a value is treated +as a credential. It does two things: + +- `isSecret()` is true for such a key on **every** cube, so a manifest that lists + `PASSWORD` in `schema` and forgets it in `secrets` still gets masking and still + keeps the value out of the session. +- The config `env` stops being broadcast for it. Ordinary `env` keys are seeded + onto every cube — deliberately, since a cube may read a key off `host.data` + that it never declared — but a secret reaches only the cubes whose + `schemaKeys()` include it. + +`declareSchema()` is what supplies those keys, and it has an ordering +requirement: call it before anything assigns to the cube, because the first +assignment is what seeds `env`. `BuildContext.resolveCube` calls it immediately +after `declareSecrets()`. It deliberately does not create the cube's bucket +itself. + `persistable()` leaves a secret out entirely rather than masking it, so a replay sees it as absent and asks for it again. That is why replaying a session whose cubes declare secrets is interactive even under `-D` — a `-D` replay that would @@ -585,15 +610,16 @@ const results = await executeDeployCalls(calls, { }); ``` -### `outputExecutionPlan(calls, asJson?)` +### `outputExecutionPlan(calls)` ```typescript -outputExecutionPlan(deployCalls); // text -outputExecutionPlan(deployCalls, true); // JSON +outputExecutionPlan(deployCalls); ``` -Both forms mask secrets. Note that `executeDeployCalls` calls this without the -second argument, so `--dry-run --json` prints the text plan. +Prints the plan a `--dry-run` shows, with secrets masked. Went from +`(calls, asJson?)` to `(calls)` when `--json` was removed; the JSON branch was +unreachable from the CLI, since `executeDeployCalls` never passed the second +argument. ### `maskCommand(call)` / `maskVariables(call)` @@ -641,7 +667,7 @@ interface WorkflowResult { authMethod: string; username?: string; password?: string; - isReplay: boolean; + replaySource?: 'file' | 'history'; // undefined on a fresh interactive run } interface WorkflowOptions { @@ -673,10 +699,12 @@ The same, from a session object rather than a path — the `-R` / `-H` path. ```typescript interface NopySession { + cubes: CubeSession[]; // required + auth: AuthSession; // required + version?: string; + timestamp?: string; // ISO 8601 name?: string; - cubes: CubeSession[]; hosts?: string[]; - auth: AuthSession; env?: TVariables; } @@ -692,7 +720,10 @@ interface AuthSession { } ``` -There is no `version` or `timestamp` field, and nothing validates compatibility. +`version` and `timestamp` are stamped on every session nopy writes and demanded +of none it reads — an older file, or a hand-written one, simply lacks them. +Nothing validates compatibility beyond a warning on an unrecognised `version`; +the constant is exported as `SESSION_VERSION`. A `CubeSession` records every value the cube settled on, whatever its origin — not just the prompted ones — minus anything the manifest declared a secret. So a @@ -702,8 +733,7 @@ and `env` happen to say later. ### `saveSession(session, filePath)` -Writes JSON, creating the directory if needed. Note that `nopy()` skips this -during a replay. +Writes JSON, creating the directory if needed. ### `loadSession(filePath)` @@ -713,7 +743,8 @@ const session = await loadSession('./deployment.session.mjs'); // default expor ``` Dispatches on the extension; `.json` and `.mjs` only. Validates that `cubes` is -an array, that `hosts` (if present) is an array, and that `auth` exists. +an array, that `hosts` (if present) is an array, and that `auth` exists. A +`version` other than `SESSION_VERSION` warns on stderr and loads anyway. ### `createSession(params)` @@ -725,18 +756,27 @@ const session = createSession({ }); ``` +Stamps `version` and `timestamp`; pass `timestamp` to override the latter. It +does not derive a `name` — that needs the resolved cube list, which does not +exist yet at the point the session is created, so `nopy()` fills it in at save +time. + +### `describeSession(session, timestamp)` + +The one-line `date - cubes → hosts` description, shared with the history list so +that the two cannot drift. + ### `listSessions(dirPath?)` -Non-recursive; matches **`*.session.json`** and **`*.session.mjs`** only. -A file named `deploy.nopysession.json` will not be listed, though `loadSession` -reads it fine. +Non-recursive; matches `*.nopysession.json`, `*.nopysession.mjs`, +`*.session.json` and `*.session.mjs`. --- ## History Module -Sessions are recorded automatically after a successful non-replay run, into -`.nopy.history.json` in the working directory. +Sessions are recorded automatically, into `.nopy.history.json` in the working +directory, before the deploy commands run — so a failed run is recorded too. ```typescript const HISTORY_FILE = '.nopy.history.json'; @@ -767,8 +807,10 @@ interface SessionHistory { | `removeFromHistory(id)` | `boolean` | `false` if the id was not found | | `formatHistoryList(entries)` | `string` | what `nopy history` prints | -Recording is suppressed for a dry run, a replay, a run that built no deploy -calls, `--no-history`, and `history.autoSave: false` in the config. +Recording is suppressed for a dry run, a print-only run, a `-R`/`-H` replay out +of history, a run that built no deploy calls, `--no-history`, and +`history.autoSave: false` in the config. A `--load-session` run **is** recorded: it is not in history already, and +without the entry `-R` would have nothing to repeat. --- @@ -785,6 +827,7 @@ interface NopyConfig { cubeDirs: string[]; cubePackages: CubePackageRef[]; env: TVariables; + secrets?: string[]; // env keys to treat as sensitive that no manifest declares log?: LogConfig; history?: HistoryConfig; execution?: ExecutionConfig; @@ -1031,7 +1074,7 @@ on `PATH`. **never throws** — it sits in front of every command the user actually asked for. Returns `null` immediately when `isUpdateCheckDisabled(env)`: `NOPY_NO_UPDATE_CHECK` set to anything but `0`/`false`, or `CI` set at all. The -CLI prints it to **stderr**, so `--json` and piped stdout stay clean. +CLI prints it to **stderr**, so a piped `--print-only` stays clean. ### `selfUpdate(options)` → `SelfUpdateResult` @@ -1062,7 +1105,6 @@ nopy install -l ./sess.json # replay a session file nopy install -n # dry run — print the plan, execute nothing nopy install -P # print the built pyinfra commands and exit nopy install -c # continue after a failure -nopy install -j # JSON output nopy install --no-history # do not record this run nopy history # list recorded sessions (alias: h; -j for JSON) @@ -1167,17 +1209,15 @@ Real behaviour that a reader would otherwise take on trust. Tracked in - **`logConfigToFlags()` is never consumed.** It is exported and unit-tested, but nothing feeds its output into the built pyinfra command, so `log.verbosity` and `log.debug` in `.nopyrc.json` have no effect today. -- **`--json` emits nothing on success.** `jsonOutput` suppresses the banner and - the progress lines, and prints `{success: false, errors}` when cube *loading* - fails. The success path returns `NopyResult` to the caller without printing it, - so a CI job gets pyinfra's inherited stdio and an exit code. `--dry-run --json` - prints the *text* plan. - **No cycle detection.** Ordering is a side effect of recursion, not a topological sort. Two mutually dependent cubes overflow the stack. - **`DeployCall.dependencies` is always `[]`.** The field is populated nowhere; dependency information lives in the emission order. - **`ExecutionResult.stdout` / `.stderr` are always `undefined`,** because the - executor inherits stdio rather than capturing it. + executor inherits stdio rather than capturing it. This is also why `install` + has no `--json`: during a run nopy does not own its own stdout, so there is no + stream to put a machine-readable answer on. Use `--print-only` for the plan and + the exit code for the verdict. - **Hook variables are not schema-validated.** The second argument to a hook is the effective values as collected. `schema.parse()` runs in exactly one place — `Cube.getDefaults()`, against `{}` — and prompt input is type-coerced, which is diff --git a/packages/nopy/docs/CUBE-PACKAGES.md b/packages/nopy/docs/CUBE-PACKAGES.md index 7c7ccb3..a86353b 100644 --- a/packages/nopy/docs/CUBE-PACKAGES.md +++ b/packages/nopy/docs/CUBE-PACKAGES.md @@ -314,8 +314,8 @@ Rename one of them, or remove a source from .nopyrc.json. There is deliberately no override, alias or precedence rule. If two bundles ever claim the same id they are mutually exclusive, and the fix is upstream. -Surface the source in the interactive picker and in `--json` output so a user can -see where a cube came from before running it. +Surface the source in the interactive picker so a user can see where a cube came +from before running it. ## Phase 4 — `@bitsquare/nopy-cubes`, the authoring package — **done** diff --git a/packages/nopy/docs/FACTS.md b/packages/nopy/docs/FACTS.md new file mode 100644 index 0000000..df60c04 --- /dev/null +++ b/packages/nopy/docs/FACTS.md @@ -0,0 +1,348 @@ +# Requirements and facts + +A proposal, not a record. Nothing here is built. + +The question it answers: a cube should be able to say *"in order to run, a user +named xyz must exist, with fish as its shell, in these groups"* — and the engine +should check that against the real host rather than blindly running whatever +cube happens to create such a user. + +Everything under [What pyinfra actually gives us](#what-pyinfra-actually-gives-us) +was measured against pyinfra **3.5.1**. Everything under +[The proposal](#the-proposal) is design. + +## Contents + +- [The problem with `dependencies`](#the-problem-with-dependencies) +- [What pyinfra actually gives us](#what-pyinfra-actually-gives-us) +- [The proposal](#the-proposal) +- [Why `facts.py` exposes a function, not a script](#why-factspy-exposes-a-function-not-a-script) +- [Execution model](#execution-model) +- [Sharp edges](#sharp-edges) +- [Measured vs. assumed](#measured-vs-assumed) + +--- + +## The problem with `dependencies` + +`manifest.dependencies` conflates two things that are not the same: + +- **a requirement** — "this cube needs a user xyz to exist" +- **a remedy** — "therefore run `user:add`" + +Today only the remedy is expressible. `user:add` declares +`dependencies: () => ['apt:essentials']`, which means *always run +`apt:essentials`*, on every host, forever, regardless of whether anything it +installs is missing. + +### What that actually costs + +Be honest about this, because it is smaller than it first looks and it changes +what the feature is for. + +pyinfra operations are **already fact-diffed**. `server.user(present=True, +shell=…)` gathers `server.Users` itself and no-ops when the state matches. The +current design is therefore not *incorrect* — it is idempotent. What it costs is: + +1. **Prompts.** Resolving `user:add` drags `apt:essentials` into the plan *and + its variables into the prompt sequence*. The user is asked questions about a + cube they never chose. This is the real, visible tax. +2. **Time.** A no-op pyinfra run is still a connection, a fact gather, and an + operation build. +3. **Legibility.** The plan cannot say *"user xyz already exists, skipping"*. + +This matters because the obvious objection to the whole feature is *"just write +`if not host.get_fact(…)` inside `deploy.py` — that is idiomatic pyinfra"*. The +answer is that nopy's layer is **planning and prompting**: deciding which cubes +enter the plan and which variables to ask for, before anything runs. A condition +inside `deploy.py` cannot help with either. That is the justification for lifting +facts into the Node runtime at all — and it means the feature buys **legibility +and prompt-avoidance, not correctness**. + +## What pyinfra actually gives us + +### The `fact` subcommand is not a machine interface + +`pyinfra @local fact server.Groups` prints JSON — and prints it to **stderr**, +via `click.echo(jsonify(…), err=True)` in `pyinfra_cli/prints.py:print_fact`, +interleaved with `--> Loading config...` progress lines. Worse, +`_run_fact_operations` wraps each fact in `except PyinfraError: pass`, so the +command **exits 0 whether or not the fact resolved**. + +Parsing that stream means fishing a JSON blob out of styled log output and +having no exit code to check. Rejected. + +### A deploy file that prints to stdout is clean + +pyinfra deploy files execute on the control machine at *build* time, and +`host.get_fact()` is available there — it is exactly how operations do their own +diffing. A file that gathers facts, prints JSON, and declares no operations +works: + +```python +import json, sys +from pyinfra import host +from pyinfra.facts.server import Users, Which + +users = host.get_fact(Users) +u = users.get(host.data.USER) +print("###NOPY-FACTS###" + json.dumps({ + "host": host.name, + "exists": u is not None, + "shell": (u or {}).get("shell"), + "groups": (u or {}).get("groups", []), +}), file=sys.stdout) +``` + +``` +$ pyinfra @local -y --data USER=someone facts.py +EXIT=0 +--- STDOUT --- +###NOPY-FACTS###{"host": "@local", "exists": false, "shell": null, "groups": []} +--- STDERR --- +--> Loading config... +… +--> Results: + Operation Hosts Success Error No Change + Grand total - - - - +``` + +Confirmed properties: + +- **stdout is exclusively ours.** Every byte pyinfra emits goes to stderr, which + is the same reason nopy's own logging goes there (`configureLogtape`). +- **`--data` flows in identically** to a deploy script. Parameterising a probe + is free. +- **Zero operations is legal.** `Grand total -`, exit 0. +- **`host.name` is available**, so multi-host output is self-tagging. +- **A custom `FactBase` subclass in a sibling module imports fine** under + `--chdir`, so a cube can ship fact classes pyinfra does not have. +- **An exception exits 1** with a traceback on stderr. + +### The trap + +**An unreachable host exits 0 and prints nothing.** + +``` +$ pyinfra nonexistent.invalid.example -y probe.py + nonexistent.invalid.example is neither an inventory file, a (list of) hosts… +EXIT=0 +--- STDOUT --- (empty) +``` + +Absence of a probe result must be a hard failure. It must never be read as +"this host has no requirements to check", which is the shape the bug would take. +The `###NOPY-FACTS###` sentinel exists for this: one line per host is *required*, +and a missing line is an error. + +### `server.Users` already covers the motivating case + +It returns `shell`, `groups`, `home`, `uid`, `gid`, `comment` and `password` per +user, keyed by name. `user:add` needs almost no custom fact code — see +[Sharp edges](#sharp-edges) for why `password` is a problem. + +## The proposal + +Split the one idea into two manifest fields, because there are two different +objects: what a cube can **report** about a host (owned by the cube responsible +for that state) and what a cube **demands** (owned by the consumer). + +A single `facts:` field cannot be both. + +### `provides` — on the cube that owns the state + +```js +// cubes/user/add/manifest.mjs +export default Manifest({ + id: 'user:add', + provides: { + /** What the probe returns. Validated on the way back in. */ + schema: z.object({ + exists: z.boolean(), + shell: z.string().nullable(), + groups: z.array(z.string()), + }), + /** Schema keys the probe needs in order to look anything up. */ + params: ['USER'], + }, + schema: z.object({ /* … unchanged … */ }), +}); +``` + +Plus a `facts.py` in the cube directory, discovered by convention exactly as +`deploy.py` is. + +### `requires` — on the consumer + +```js +requires: (vars) => [{ + cube: 'user:add', + with: { USER: vars.DEPLOY_USER }, + expect: z.object({ + exists: z.literal(true), + shell: z.literal('/usr/bin/fish'), + groups: z.array(z.string()).refine((g) => g.includes('docker')), + }), +}], +``` + +Three deliberate choices: + +**A requirement names its provider.** If `requires` stated only a predicate, the +engine would have to search the cube space for something that satisfies an +arbitrary Zod schema. That is a planner, and a planner is a research project. +Naming the cube keeps resolution linear and keeps the failure message legible. + +**`requires` is `(vars) => …`, like `dependencies` already is.** A requirement +almost always depends on the consumer's own variables — you cannot know *which* +user to check for until the consumer has been asked. + +**Zod is the predicate language.** It is already this repo's schema vocabulary, +`z.literal` and `.refine()` cover the cases, and `z.treeifyError()` produces the +failure report for free. The cost is that `.refine()` closures are not +serialisable, so a requirement can never be written into a session — only its +*result* can. See [Sharp edges](#sharp-edges). + +### The engine's rule + +This is the half of the design that "only run if requirements are fulfilled" +leaves out. When a requirement is **not** met, there are three possible answers: + +| | | +| --- | --- | +| **Abort** | Honest, and useless. On a bare host nothing is fulfilled, so every fresh deploy fails. | +| **Skip the cube** | A silent no-op. Dangerous. | +| **Remedy** | Run the named cube. | + +It has to be *remedy* — and remedy means "run the cube that provides it", which +is `dependencies` again. That is the actual insight here, and it is a much +smaller change than a parallel subsystem: + +> **You do not need a new mechanism. You need dependencies to become conditional.** + +So: + +- requirement **met** → the provider is *not* scheduled and its variables are + *not* prompted for. This is where the prompt tax disappears. +- requirement **unmet** → the provider is scheduled with `with` applied as + overrides. `with` assigns at origin `param`, which already outranks every + other origin (`default < env < session < prompt < param`), so no new + precedence rule is needed. + +## Why `facts.py` exposes a function, not a script + +The obvious design is symmetry: `deploy.py` is a script, so `facts.py` is a +script. Reject it. + +A recursive resolution over a dependency tree issues one probe per (cube, host, +params). At one pyinfra invocation each, that is one SSH connection each — +seconds apiece, multiplied by the tree. Unaffordable. + +But probes are **pure reads with no ordering constraints between them**, which +means every probe for a given host can be gathered in a *single* pyinfra +invocation: one generated driver script that imports each cube's `facts.py`, +calls it with its params, and emits one JSON object per host. One connection per +host per resolution round, instead of one per cube. + +A standalone script can only be run alone. A function can be batched: + +```python +# cubes/user/add/facts.py +from pyinfra.facts.server import Users + +def gather(host, params): + u = host.get_fact(Users).get(params["USER"]) + return { + "exists": u is not None, + "shell": (u or {}).get("shell"), + "groups": (u or {}).get("groups", []), + } +``` + +Consequence: params arrive as one `--data NOPY_PROBES=` blob rather than +per-cube `--data KEY=…`, since a batched run carries several cubes' params at +once. Probe results are memoised per (cube, host, params) in the `BuildContext`, +the same shape as the existing `resolvedCubes` set. + +## Execution model + +Probes run during **resolution**, before anything has deployed. That has a +consequence worth stating plainly rather than discovering later: + +When the plan reads *"user absent → schedule `user:add` → then B"*, B's +requirement was never verified. It was **assumed** met because a remedy was +scheduled. Every tool in this space stops there. + +Since the probe already exists, the loop can be closed for nearly nothing: + +> **Re-run the provider's probe after it deploys, and fail loudly if the +> requirement still is not met.** + +`facts.py` then serves as a post-condition test as well as a precondition check. +This is the single most valuable property of the design — it is strictly more +than Ansible's `when:` offers — and it should be built in from the start rather +than added as a later refinement. + +Sketch of the resolution change in `BuildContext.resolveCube`, between steps 3 +(`before` hooks) and 4 (dependencies): + +1. Collect `manifest.requires?.(currentVars)`. +2. Batch every unmet-so-far probe for this host into one pyinfra run. +3. Parse each result against the provider's `provides.schema`, then against the + consumer's `expect`. +4. For each failure, `resolveCube(req.cube, host, req.with)`. +5. After that provider's deploy call executes, re-probe and assert. + +Step 5 does not fit the current shape: `deployCalls` are all built first and +executed later, so a post-condition needs the executor to call back into +probing. That is the one structurally invasive part of this proposal. + +## Sharp edges + +**`--print-only` purity.** A dry run touches no host today. Probing breaks that +outright. Needs an explicit answer — either a probe-free plan that renders +requirements as unevaluated, or an opt-in flag. Silently connecting during what +the user believes is a dry run is not acceptable. + +**Secret leakage.** `server.Users` returns the **encrypted password hash** in +its `password` field. A probe result that reaches history, a plan dump, or a log +line is a credential leak. The existing `secrets` machinery is per-*variable* +and does not apply — facts need their own redaction rule. Easy to miss, hard to +take back. + +**Sessions must never persist facts.** Facts are host state at a moment in time; +a replay must re-probe, never restore. Recording them in *history* for +diagnostics is fine and useful. This is also forced by `.refine()` being +unserialisable. + +**Sudo.** Many useful facts require `_sudo`, and a probe runs before the deploy +has made any of its own auth decisions. Unresolved — worth a spike. + +**Cycles.** `requires` introduces a second edge type into a graph that still has +no cycle detection (see *Known drift* in `CLAUDE.md`). Conditional edges make +"A requires B, B requires A under different params" considerably more reachable +than the current unconditional ones do. + +**Naming.** `provides` / `requires` over the original `facts:`, because the +field names should say which side of the relationship they belong to. + +## Measured vs. assumed + +Measured against pyinfra 3.5.1, on this machine: + +- `fact` subcommand writes JSON to stderr and exits 0 on fact failure +- a deploy file's stdout is uncontaminated by pyinfra output +- `--data` reaches a probe as `host.data.KEY` +- a deploy file with zero operations succeeds +- a custom `FactBase` in a sibling module resolves under `--chdir` +- an exception in a deploy file exits 1 +- **an unreachable host exits 0 with empty stdout** + +Assumed, not yet tested: + +- batching several cubes' probes into one driver script works and is meaningfully + cheaper than N invocations — this is the load-bearing cost assumption and + should be the first thing spiked +- `host.get_fact()` accepts `_sudo` from within a probe function +- the post-condition re-probe can be threaded through `executeDeployCalls` + without unpicking the build-then-execute split diff --git a/packages/nopy/docs/SESSION_FORMAT.md b/packages/nopy/docs/SESSION_FORMAT.md index 23f6247..adfe8c2 100644 --- a/packages/nopy/docs/SESSION_FORMAT.md +++ b/packages/nopy/docs/SESSION_FORMAT.md @@ -2,9 +2,14 @@ Nopy supports two session file formats: **JSON** and **MJS** (ES Module JavaScript). +The extension is what picks the loader, so a session file has to end in `.json` +or `.mjs`; anything else is refused by name. The `.nopysession.*` names used +throughout are the convention `listSessions()` looks for — `-s` and `-l` accept +any path you give them. + ## Supported Formats -### JSON Format (`.session.json`) +### JSON Format (`.nopysession.json`) Traditional JSON format for session files: @@ -35,7 +40,7 @@ Traditional JSON format for session files: - Cannot use dynamic values or computation - No code reuse or imports -### MJS Format (`.session.mjs`) - **Recommended** +### MJS Format (`.nopysession.mjs`) - **Recommended** JavaScript module format with full ES Module support: @@ -172,7 +177,7 @@ export const commonCubes = [ ]; ``` -**my-session.session.mjs:** +**my-session.nopysession.mjs:** ```javascript import { productionHosts, commonCubes } from './common-config.mjs'; @@ -232,7 +237,7 @@ function generateSession(config) { }; const content = `export default ${JSON.stringify(session, null, 2)};`; - fs.writeFileSync('generated.session.mjs', content); + fs.writeFileSync('generated.nopysession.mjs', content); } // Generate from external configuration @@ -255,10 +260,10 @@ Both formats are loaded the same way: import { loadSession } from '@bitsquare/nopy'; // Load JSON -const jsonSession = await loadSession('./my-session.session.json'); +const jsonSession = await loadSession('./my-session.nopysession.json'); // Load MJS -const mjsSession = await loadSession('./my-session.session.mjs'); +const mjsSession = await loadSession('./my-session.nopysession.mjs'); ``` The file extension determines which loader to use. @@ -267,7 +272,7 @@ The file extension determines which loader to use. To convert an existing JSON session to MJS: -1. Rename the file from `.session.json` to `.session.mjs` +1. Rename the file from `.nopysession.json` to `.nopysession.mjs` 2. Add `export default` before the configuration object 3. Remove quotes from property keys (optional) 4. Add comments and dynamic values as needed @@ -304,11 +309,12 @@ Both formats must export/contain an object with this structure: ```typescript interface NopySession { - version: string; // Session format version - timestamp: string; // ISO timestamp - cubes: CubeSession[]; // Array of cube configurations - hosts: string[]; // Target hosts - auth: AuthSession; // Authentication configuration + cubes: CubeSession[]; // Array of cube configurations — required + auth: AuthSession; // Authentication configuration — required + version?: string; // Session format version, currently "1.0.0" + timestamp?: string; // ISO timestamp + name?: string; // One-line description + hosts?: string[]; // Target hosts env?: Record; // Global environment variables } @@ -323,6 +329,17 @@ interface AuthSession { } ``` +Only `cubes` and `auth` are demanded of a session being *read* — the loader +requires what it cannot work without and nothing else, so the sessions in these +examples are all valid, and one written before `version` existed still loads. A +session nopy *writes* always carries `version`, `timestamp` and `name`; a +`version` this build does not recognise produces a warning on stderr and loads +anyway. + +`method: 'ssh'` is the third value and the one no prompt produces: it means the +connector handles authentication and nopy supplies no credential. Every +`@vagrant/` and `@docker/` host gets it. + A session nopy *writes* holds, per cube, every value that cube ran with — what was typed, what came from `.nopyrc.json`, what a dependency supplied, and what fell through to the schema's `.default()`. Two things are deliberately absent and diff --git a/packages/nopy/docs/VAGRANT.md b/packages/nopy/docs/VAGRANT.md index f402663..42cced8 100644 --- a/packages/nopy/docs/VAGRANT.md +++ b/packages/nopy/docs/VAGRANT.md @@ -1,7 +1,9 @@ # Vagrant -`vagrant ssh-config` to find the SSH port of the machine -`vagrant status --machine-readable` will be executed by pyinfra to get information about available VMs +A Vagrant box is the cheapest way to run a cube against a real machine you can +throw away afterwards. + +## The Vagrantfile ```ruby @@ -19,3 +21,34 @@ Vagrant.configure("2") do |config| end ``` + +## Naming the machine to nopy + +The host string is `@vagrant/`, where `` is what `config.vm.define` +declared — `nopytestvm` above. It is pyinfra's connector syntax, not nopy's, and +it is the same shape as `@docker/`. + +Two ways to get there. Either pick `vagrant` in the host prompt and answer the +follow-up with the machine name, which is what builds the string for you, or put +it in `.nopyrc.json` so it appears in the list directly: + +```json +{ + "hosts": ["@vagrant/nopytestvm"], + "cubePackages": ["@bitsquare/nopy-cubes-core"] +} +``` + +pyinfra runs `vagrant status --machine-readable` to find the available machines +and `vagrant ssh-config` for the SSH port, so `vagrant` has to be on `PATH` and +the box has to be `up` before a deploy. + +## Cleaning up + +```sh +vagrant halt # stop it, keep the disk +vagrant destroy -f # delete it — the next `vagrant up` is a fresh box +``` + +`destroy` is the one to use between test runs of a cube that is not idempotent: +re-running against a half-configured box tests something other than the cube. diff --git a/packages/nopy/example.nopysession.json b/packages/nopy/example.nopysession.json index 32dc148..ce95d1c 100644 --- a/packages/nopy/example.nopysession.json +++ b/packages/nopy/example.nopysession.json @@ -1,5 +1,7 @@ { + "version": "1.0.0", "name": "Example Deployment Session", + "timestamp": "2026-07-30T09:15:00.000Z", "cubes": [ { "key": "apt:essentials", diff --git a/packages/nopy/src/cubes/dependencies.ts b/packages/nopy/src/cubes/dependencies.ts index f439658..c1d8162 100644 --- a/packages/nopy/src/cubes/dependencies.ts +++ b/packages/nopy/src/cubes/dependencies.ts @@ -7,6 +7,7 @@ import type { Cube, CubeVariables, HookContext } from '@bitsquare/nopy-cubes'; import { getLogger } from '@logtape/logtape'; import type { Variables } from '../nopy.common.js'; import type { NopyConfig } from '../nopy.config.js'; +import { NopyUsageError } from '../nopy.errors.js'; import type { DeployCall } from '../nopy.executor.js'; import { VariableAssignment } from '../nopy.prompts.js'; import type { CubeSession, NopySession } from '../nopy.session.js'; @@ -45,21 +46,33 @@ export class BuildContext { } /** - * Fails a non-interactive run that cannot fill a required variable. + * Fails a run that cannot fill a required variable. * * Without this the cube would be deployed with the key simply absent from - * `--data`, and the deploy script would read `None` off `host.data`. + * `--data`, and the deploy script would read `None` off `host.data` — against + * the documented guarantee that every schema key reaches it. + * + * Runs on the interactive path too, not only under `--use-defaults`. A prompt + * is not proof of an answer: a terminal that misreports its size renders an + * empty form and submits `{}` without the user seeing a field, which is + * exactly how this was found. */ private assertVariablesComplete(cube: Cube): void { const missing = this.missingRequired(cube); if (missing.length === 0) return; - const [one, them] = - missing.length === 1 ? ['has no default value', 'it'] : ['have no default values', 'them']; - throw new Error( - `Cube "${cube.id}" cannot run with --use-defaults: ${missing.join(', ')} ${one}. ` + - `Set ${them} under "env" in .nopyrc.json, pass ${them} from a dependency, ` + - 'or drop --use-defaults to be prompted.' + const list = missing.join(', '); + const them = missing.length === 1 ? 'it' : 'them'; + const have = missing.length === 1 ? 'has no default value' : 'have no default values'; + + throw new NopyUsageError( + this.options.useDefaults + ? `Cube "${cube.id}" cannot run with --use-defaults: ${list} ${have}. ` + + `Set ${them} under "env" in .nopyrc.json, pass ${them} from a dependency, ` + + 'or drop --use-defaults to be prompted.' + : `Cube "${cube.id}" is missing ${list}. Nothing supplied ${them} — the form may have ` + + `been submitted empty. Re-run and fill ${them} in, or set ${them} under "env" ` + + 'in .nopyrc.json.' ); } @@ -81,23 +94,39 @@ export class BuildContext { if (gaps.length === 0) return; if (this.options.useDefaults) { - throw new Error( - `Cube "${cube.id}" cannot be replayed with --use-defaults: ${gaps.join(', ')} ` + - 'would have to be entered. Secrets are never recorded in a session. ' + - 'Replay without --use-defaults, or set the values under "env" in .nopyrc.json.' - ); + // A gap is only a gap if nothing outside the session filled it. `env` and + // `param` both say deliberately what the value is, which is exactly what + // the old message told the user to do — and then failed anyway. + // + // `default` is not accepted here. The session dropped the secret on + // purpose, so falling through to a manifest default would deploy a + // different credential than the run being replayed, without saying so. + const unsatisfied = gaps.filter((key) => { + const origin = this.variables.of(cube.id, key)?.origin; + return origin !== 'env' && origin !== 'param'; + }); + + if (unsatisfied.length > 0) { + const them = unsatisfied.length === 1 ? 'it' : 'them'; + const secret = unsatisfied.some((key) => cube.secrets.includes(key)); + throw new NopyUsageError( + `Cube "${cube.id}" cannot be replayed with --use-defaults: ` + + `${unsatisfied.join(', ')} would have to be entered. ` + + (secret ? 'Secrets are never recorded in a session. ' : '') + + `Set ${them} under "env" in .nopyrc.json` + + (secret ? ' (a schema default is not accepted for a secret)' : '') + + `, pass ${them} from a dependency, or replay without --use-defaults.` + ); + } + return; } log.debug('Filling session gaps', { cubeId: cube.id, gaps }); await VariableAssignment(cube, this.variables, { keys: gaps }); - // A cancelled form leaves the run short of a value it cannot invent. - const stillMissing = this.missingRequired(cube); - if (stillMissing.length > 0) { - throw new Error( - `Cube "${cube.id}" is missing ${stillMissing.join(', ')} and cannot be deployed.` - ); - } + // A form that resolved is not a form that was answered — same check, and + // the same reason for it, as the interactive path. + this.assertVariablesComplete(cube); } /** @@ -110,15 +139,19 @@ export class BuildContext { ): Promise { const cube = this.allCubes[cubeId]; if (!cube) { - throw new Error(`Cube not found: ${cubeId}`); + throw new NopyUsageError(`Cube not found: ${cubeId}`); } log.debug('Resolving cube', { cubeId, host }); - // 1. Declare secrets, then assign overrides and defaults. Declaring first - // means even the config `env` seeded on the cube's first assignment is - // already marked, so nothing reaches a session or a log unredacted. + // 1. Declare secrets and schema, then assign overrides and defaults. Both + // declarations have to come first: the cube's first assignment is what + // seeds the config `env` onto it, and by then it must already be known + // which of those keys are secret (so nothing reaches a session or a log + // unredacted) and which the cube actually declares (so a secret it does + // not declare is never seeded at all). this.variables.declareSecrets(cubeId, cube.secrets); + this.variables.declareSchema(cubeId, cube.schemaKeys()); if (Object.keys(overrides).length > 0) { this.variables.assign(cubeId, 'param', overrides); } @@ -136,6 +169,7 @@ export class BuildContext { this.assertVariablesComplete(cube); } else { await VariableAssignment(cube, this.variables); + this.assertVariablesComplete(cube); } const currentVars = this.variables.get(cubeId); diff --git a/packages/nopy/src/nopy.cli.ts b/packages/nopy/src/nopy.cli.ts index 7be9bdc..d21e494 100644 --- a/packages/nopy/src/nopy.cli.ts +++ b/packages/nopy/src/nopy.cli.ts @@ -8,6 +8,7 @@ import { createRequire } from 'node:module'; import { Command } from 'commander'; import { loadConfig } from './nopy.config.js'; +import { reportError } from './nopy.errors.js'; import { exitWithFarewell, installGracefulExit, isCancellation } from './nopy.exit.js'; import { clearHistory, @@ -33,8 +34,8 @@ const { version, buildInfo } = createRequire(import.meta.url)('../package.json') const versionLabel = buildInfo?.commit ? `${version} (${buildInfo.commit})` : version; /** - * Prints the update hint to stderr, so it never lands in `--json` output or in - * a `--print-only` command list being piped somewhere. + * Prints the update hint to stderr, so it never lands in a `--print-only` + * command list being piped somewhere. */ async function printUpdateNotice(): Promise { const notice = await updateNotice({ currentVersion: version }); @@ -67,6 +68,9 @@ Examples: $ nopy history List all saved sessions $ nopy clear-history Clear session history + Every flag above belongs to 'install', the default command — 'nopy -R' is + 'nopy install -R'. Run 'nopy install --help' for the full list. + Session Replay: Sessions are automatically saved to history after each deployment. Use 'nopy history' to see available sessions and their IDs. @@ -88,16 +92,19 @@ program .option('-n, --dry-run', 'Show execution plan without running') .option('-P, --print-only', 'Print deploy commands and exit (no execution)') .option('-c, --continue-on-error', 'Continue executing after failures') - .option('-j, --json', 'Output results as JSON') .option('--no-history', 'Do not save this session to history') .action(async (options) => { await printUpdateNotice(); - // Loaded lazily so that --help/--version work outside a configured project. - const execConfig = loadConfig().execution ?? {}; - const continueOnError = options.continueOnError ?? execConfig.continueOnError ?? false; - try { + // Loaded lazily so that --help/--version work outside a configured + // project — and inside the try, so that "no .nopyrc.json here" is + // reported by `reportError` rather than escaping as an unhandled + // rejection and printing node's own stack. It is the likeliest first-run + // mistake there is. + const execConfig = loadConfig().execution ?? {}; + const continueOnError = options.continueOnError ?? execConfig.continueOnError ?? false; + // Handle session replay const loadSessionPath = options.loadSession; let sessionToReplay: { session: import('./nopy.session.js').NopySession } | undefined; @@ -109,7 +116,9 @@ program process.exit(1); } sessionToReplay = lastEntry; - console.log(`Repeating: ${lastEntry.name}\n`); + // stderr, like everything nopy says about itself — `-R --print-only` has + // to leave stdout to the commands. + console.error(`Repeating: ${lastEntry.name}\n`); } else if (options.history) { const entry = getSessionById(options.history); if (!entry) { @@ -118,7 +127,7 @@ program process.exit(1); } sessionToReplay = entry; - console.log(`Running: ${entry.name}\n`); + console.error(`Running: ${entry.name}\n`); } const result = await nopy({ @@ -130,7 +139,6 @@ program dryRun: options.dryRun, printOnly: options.printOnly, continueOnError, - jsonOutput: options.json, saveToHistory: options.history !== false && !options.dryRun, }); @@ -144,20 +152,7 @@ program // the process-level handler. if (isCancellation(error)) exitWithFarewell(); - if (options.json) { - console.log( - JSON.stringify( - { - success: false, - error: error instanceof Error ? error.message : String(error), - }, - null, - 2 - ) - ); - } else { - console.error('Error:', error instanceof Error ? error.message : error, error); - } + reportError(error); process.exit(1); } }); diff --git a/packages/nopy/src/nopy.common.ts b/packages/nopy/src/nopy.common.ts index d845238..fb24185 100644 --- a/packages/nopy/src/nopy.common.ts +++ b/packages/nopy/src/nopy.common.ts @@ -114,8 +114,21 @@ export class Variable { export class Variables { private readonly store: Record> = {}; private readonly secrets: Record> = {}; + private readonly schemas: Record> = {}; + private readonly globalSecrets: Set; - constructor(readonly env: TVariables = {}) {} + /** + * @param env - the `env` block of the merged config, seeded onto every cube + * @param globalSecrets - every key *any* manifest declares secret, plus the + * config's own `secrets` list. Known up front, before the first cube + * resolves, so it does not depend on resolution order. + */ + constructor( + readonly env: TVariables = {}, + globalSecrets: Iterable = [] + ) { + this.globalSecrets = new Set(globalSecrets); + } /** * Marks keys of one cube as holding secrets. @@ -132,8 +145,30 @@ export class Variables { } } + /** + * Records which keys a cube's schema declares. + * + * Only {@link bucket} reads this, and only to decide whether a globally + * declared secret may be seeded from `env`. Call it before anything assigns to + * the cube — it deliberately does not create the bucket itself, because + * creating it is what seeds `env`. + */ + declareSchema(cube: string, keys: readonly string[]): void { + this.schemas[cube] ??= new Set(); + const declared = this.schemas[cube]; + for (const key of keys) declared.add(key); + } + + /** + * Whether a key is sensitive for a cube. + * + * True for a key the cube's own manifest declared, and also for one *another* + * manifest declared: a value that is a secret anywhere is a secret everywhere + * it lands. That covers the manifest that lists `PASSWORD` in `schema` and + * forgets it in `secrets`. + */ isSecret(cube: string, name: string): boolean { - return this.secrets[cube]?.has(name) ?? false; + return (this.secrets[cube]?.has(name) ?? false) || this.globalSecrets.has(name); } /** Records values for one cube, all at the same origin. */ @@ -176,6 +211,24 @@ export class Variables { return values; } + /** + * The config's `env` block minus anything declared secret — what a session's + * own `env` records. + * + * A session copies `env` verbatim for reference, which quietly undid + * {@link persistable}: a credential declared in `.nopyrc.json` was kept out of + * every cube's `variables` and then written to the same file one key higher up, + * in plaintext, along with a copy in `.nopy.history.json`. Same rule as + * `persistable`, applied to the same file. + */ + persistableEnv(): TVariables { + const values: TVariables = {}; + for (const [name, value] of Object.entries(this.env)) { + if (!this.globalSecrets.has(name)) values[name] = value; + } + return values; + } + private create(cube: string, name: string, first: Assignment): Variable { const variable = new Variable(cube, name, first); variable.redacted = this.isSecret(cube, name); @@ -189,6 +242,12 @@ export class Variables { * rather than a parallel bag merged in at read time. That is what lets it * carry an origin, show up in the trace, and lose to a prompt by the same rule * as everything else. + * + * One key is held back: a **secret**, on a cube whose schema does not mention + * it. Broadcasting is otherwise load-bearing — a cube may legitimately read a + * key off `host.data` that it never declared — but a credential does not + * belong on the command line of every unrelated cube in the run, where nothing + * masks it because that cube never declared it sensitive. */ private bucket(cube: string): Record { const existing = this.store[cube]; @@ -197,6 +256,7 @@ export class Variables { const bucket: Record = {}; this.store[cube] = bucket; for (const [name, value] of Object.entries(this.env)) { + if (this.globalSecrets.has(name) && !this.schemas[cube]?.has(name)) continue; bucket[name] = this.create(cube, name, { value, origin: 'env' }); } return bucket; diff --git a/packages/nopy/src/nopy.config.ts b/packages/nopy/src/nopy.config.ts index 16a13b5..11d5945 100644 --- a/packages/nopy/src/nopy.config.ts +++ b/packages/nopy/src/nopy.config.ts @@ -6,6 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; import type { TVariables } from './nopy.common.js'; +import { NopyUsageError } from './nopy.errors.js'; /** * Log verbosity levels for pyinfra output @@ -102,6 +103,14 @@ export interface NopyConfig { cubePackages: CubePackageRef[]; /** Global environment variables */ env: TVariables; + /** + * `env` keys to treat as sensitive even though no manifest says so. + * + * A manifest's own `secrets` list already covers the cubes that declare the + * key. This is for the value no cube declares at all — a token a hook reads, + * say — which would otherwise be broadcast and printed in the clear. + */ + secrets?: string[]; /** Logging configuration */ log?: LogConfig; /** Session history configuration */ @@ -317,7 +326,7 @@ export function loadConfig(): NopyConfig { const configPaths = findConfigFiles(); if (configPaths.length === 0) { - throw new Error( + throw new NopyUsageError( `No ${CONFIG_FILENAME} found. Create one in your project directory or any parent directory.` ); } @@ -334,7 +343,7 @@ export function loadConfig(): NopyConfig { config = mergeConfigs(config, resolvedConfig); } catch (err) { const message = err instanceof Error ? err.message : String(err); - throw new Error(`Failed to load config ${configPath}: ${message}`); + throw new NopyUsageError(`Failed to load config ${configPath}: ${message}`); } } diff --git a/packages/nopy/src/nopy.errors.ts b/packages/nopy/src/nopy.errors.ts new file mode 100644 index 0000000..1f19938 --- /dev/null +++ b/packages/nopy/src/nopy.errors.ts @@ -0,0 +1,50 @@ +/** + * The errors that are the user's to fix. + * @module nopy.errors + */ + +/** + * A run that failed for a reason the user can act on: no config file, a cube + * that does not exist, a required variable nothing supplied, a session file + * that will not load. + * + * The point is the *presentation*, not the control flow — nothing catches this + * to recover. A stack trace through `dist/` says nothing useful about a missing + * `.nopyrc.json`, and printing one invites the reader to look for a bug in nopy + * instead of a typo in their project. The CLI prints the message alone and keeps + * the stack behind `NOPY_DEBUG`. + * + * Mirrors keyman's `UsageError` deliberately: the two CLIs are kept in step on + * how they fail for the same reason their update modules are duplicated rather + * than shared. + */ +export class NopyUsageError extends Error { + constructor(message: string) { + super(message); + this.name = 'NopyUsageError'; + } +} + +/** + * Reports a failed run in as many lines as it deserves. + * + * A {@link NopyUsageError} prints as one line: it is something the reader can + * fix, and three frames into `dist/` say nothing about a missing `.nopyrc.json` + * except that it looks like a crash in nopy rather than a typo in the project. + * Everything else keeps its stack, because an unexpected failure is exactly when + * one is worth having. `NOPY_DEBUG` forces it for both. + * + * Lives here rather than in `nopy.cli.ts` because the CLI is excluded from + * coverage — it is argv wiring, and this is a decision. + */ +export function reportError(error: unknown): void { + const message = error instanceof Error ? error.message : String(error); + + console.error(`Error: ${message}`); + + const stack = error instanceof Error ? error.stack : undefined; + const wanted = process.env.NOPY_DEBUG || !(error instanceof NopyUsageError); + + if (wanted && stack) console.error(stack); + else if (!process.env.NOPY_DEBUG) console.error('Set NOPY_DEBUG=1 for the full stack trace.'); +} diff --git a/packages/nopy/src/nopy.executor.ts b/packages/nopy/src/nopy.executor.ts index 4d80f36..8ac4efa 100644 --- a/packages/nopy/src/nopy.executor.ts +++ b/packages/nopy/src/nopy.executor.ts @@ -143,20 +143,8 @@ async function executeCall(call: DeployCall): Promise { * Outputs the execution plan without running (dry run) * * @param calls - Array of deployment calls - * @param asJson - Output as JSON instead of text */ -export function outputExecutionPlan(calls: DeployCall[], asJson?: boolean): void { - if (asJson) { - const plan = calls.map((call) => ({ - cube: call.cube, - host: call.host, - command: maskCommand(call), - variables: maskVariables(call), - })); - console.log(JSON.stringify({ plan }, null, 2)); - return; - } - +export function outputExecutionPlan(calls: DeployCall[]): void { console.log('\n=== Execution Plan (Dry Run) ===\n'); for (let i = 0; i < calls.length; i++) { diff --git a/packages/nopy/src/nopy.exit.ts b/packages/nopy/src/nopy.exit.ts index 89daa08..fc4daec 100644 --- a/packages/nopy/src/nopy.exit.ts +++ b/packages/nopy/src/nopy.exit.ts @@ -74,7 +74,7 @@ export function restoreTerminal(): void { * Says goodbye and leaves. * * The farewell goes to **stderr**, for the same reason the update hint does: - * `--json` and `--print-only` stay machine-readable no matter how the run ends. + * `--print-only` stays machine-readable no matter how the run ends. * * `process.exit` rather than letting the loop drain, because the prompt that * was cancelled is still holding stdin — after the teardown above threw, its diff --git a/packages/nopy/src/nopy.history.ts b/packages/nopy/src/nopy.history.ts index 7167610..a6709d9 100644 --- a/packages/nopy/src/nopy.history.ts +++ b/packages/nopy/src/nopy.history.ts @@ -5,7 +5,7 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { NopySession } from './nopy.session.js'; +import { describeSession, type NopySession } from './nopy.session.js'; /** Default number of sessions to keep in history */ export const DEFAULT_HISTORY_SIZE = 10; @@ -72,35 +72,6 @@ export function saveHistory(history: SessionHistory): void { fs.writeFileSync(historyPath, JSON.stringify(history, null, 2), 'utf-8'); } -/** - * Generates a history entry name from session data - * - * Format: "YYYY-MM-DD HH:mm - cube1, cube2, ..." - * - * @param session - The session to name - * @param timestamp - ISO timestamp - * @returns Human-readable name - */ -function generateEntryName(session: NopySession, timestamp: string): string { - const date = new Date(timestamp); - const dateStr = date.toLocaleString('en-US', { - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - hour12: false, - }); - - const cubeNames = session.cubes.map((c) => c.key).join(', '); - const truncatedCubes = cubeNames.length > 40 ? `${cubeNames.substring(0, 37)}...` : cubeNames; - - const hosts = session.hosts?.join(', ') || 'no host'; - const truncatedHosts = hosts.length > 20 ? `${hosts.substring(0, 17)}...` : hosts; - - return `${dateStr} - ${truncatedCubes} → ${truncatedHosts}`; -} - /** * Generates a unique ID for a history entry */ @@ -124,7 +95,7 @@ export function addToHistory( const entry: HistoryEntry = { id: generateEntryId(), - name: generateEntryName(session, timestamp), + name: describeSession(session, timestamp), timestamp, session, }; diff --git a/packages/nopy/src/nopy.main.ts b/packages/nopy/src/nopy.main.ts index fad4ff9..22e4725 100644 --- a/packages/nopy/src/nopy.main.ts +++ b/packages/nopy/src/nopy.main.ts @@ -15,11 +15,16 @@ import { summarizeResults, } from './nopy.executor.js'; import { addToHistory, DEFAULT_HISTORY_SIZE } from './nopy.history.js'; -import { type NopySession, saveSession } from './nopy.session.js'; +import { describeSession, type NopySession, SESSION_VERSION, saveSession } from './nopy.session.js'; import { runWorkflow } from './nopy.workflow.js'; /** - * Configures the logtape logger for console output + * Configures the logtape logger for console output. + * + * **stderr**, deliberately. stdout carries the deploy commands and pyinfra's own + * output; everything nopy says about itself goes to stderr, so `--print-only` + * can be piped somewhere. The sink used to write to stdout and was held back + * only by `--json`, which never worked and is gone. */ function configureLogtape(): void { configure({ @@ -31,7 +36,7 @@ function configureLogtape(): void { if (typeof formatted === 'string') { const msg = formatted.replace(/\r?\n$/, ''); const props = record.properties as Record; - console.log(msg, ...Object.values(props)); + console.error(msg, ...Object.values(props)); } }; })(), @@ -55,7 +60,7 @@ function configureLogtape(): void { configureLogtape(); /** - * Prints the active configuration summary + * Prints the active configuration summary — to stderr, see {@link configureLogtape}. */ function printActiveConfig( config: import('./nopy.config.js').NopyConfig, @@ -92,7 +97,7 @@ function printActiveConfig( } lines.push(''); - console.log(lines.join('\n')); + console.error(lines.join('\n')); } /** @@ -107,7 +112,6 @@ export interface NopyOptions { dryRun?: boolean; printOnly?: boolean; continueOnError?: boolean; - jsonOutput?: boolean; saveToHistory?: boolean; } @@ -138,24 +142,30 @@ export async function nopy(opts: NopyOptions = {}): Promise cube.secrets), + ...(config.secrets ?? []), + ]); + const variables = new Variables(config.env, declaredSecrets); if (errors.length > 0) { log.error('Errors found during cube loading:'); for (const error of errors) log.error(error); - if (jsonOutput) console.log(JSON.stringify({ success: false, errors }, null, 2)); return undefined; } @@ -180,7 +190,7 @@ export async function nopy(opts: NopyOptions = {}): Promise 0) { + // A `-R`/`-H` replay is already in history and re-recording it would push the + // original out of the list. A `--load-session` run is not in history at all, + // so unless it is recorded here, `nopy history` reports nothing afterwards and + // `-R` has nothing to repeat. + const recordable = workflow.replaySource !== 'history'; + + // `--print-only` is excluded for the same reason `--dry-run` is: neither + // deployed anything, and history is what `-R` repeats. Recording a run that + // never happened made `nopy install -P` — the safe look-before-you-leap flag — + // silently displace the last real deployment at the head of the list. + if (saveToHistory && !dryRun && !printOnly && recordable && context.deployCalls.length > 0) { const historySize = config.history?.maxSessions ?? DEFAULT_HISTORY_SIZE; if (config.history?.autoSave !== false) { addToHistory(sessionForSaving, historySize); @@ -224,10 +260,8 @@ export async function nopy(opts: NopyOptions = {}): Promise { - if (!jsonOutput) { - const status = result.success ? '✓' : '✗'; - log.info(`[${completed}/${total}] ${status} ${result.cube} -> ${result.host}`); - } + const status = result.success ? '✓' : '✗'; + log.info(`[${completed}/${total}] ${status} ${result.cube} -> ${result.host}`); }, }); diff --git a/packages/nopy/src/nopy.prompts.ts b/packages/nopy/src/nopy.prompts.ts index 88d4a57..a15b226 100644 --- a/packages/nopy/src/nopy.prompts.ts +++ b/packages/nopy/src/nopy.prompts.ts @@ -17,6 +17,45 @@ interface CubeChoice { message: string; } +/** Floor for a terminal that reports a size no prompt could render into. */ +const MIN_ROWS = 24; +const MIN_COLS = 80; + +/** + * The window size to hand an enquirer prompt, never smaller than {@link MIN_ROWS}. + * + * Load-bearing, not cosmetic. enquirer derives how many choices are visible from + * its height, and `utils.height` (`lib/utils.js:80`) computes a sane fallback and + * then throws it away: + * + * ```js + * let rows = (stream && stream.rows) ? stream.rows : fallback; + * if (stream && typeof stream.getWindowSize === 'function') { + * rows = stream.getWindowSize()[1]; // unconditional + * } + * ``` + * + * A TTY always has `getWindowSize`, so a terminal reporting 0 rows — some CI + * pseudo-terminals, `script -q`, an editor terminal mid-startup — yields + * `height: 0`, `Math.min(limit, 0)` choices, and a form that renders nothing and + * submits `{}`. Passing `rows` bypasses that: `prompt.js:396` reads + * `this.options.rows || utils.height(...)`, so the broken function never runs. + * + * Measured on a 0×0 pty: without this the four-field form returns `{}`; with it, + * every field. No effect on a terminal that reports its size honestly. + * enquirer 2.4.1 is its final release, so the bug is not going to be fixed + * upstream. + */ +function terminalSize(out: NodeJS.WriteStream = process.stdout): { + rows: number; + columns: number; +} { + return { + rows: Math.max(out.rows || 0, MIN_ROWS), + columns: Math.max(out.columns || 0, MIN_COLS), + }; +} + /** * Fuzzy-filters the cube list against what the user has typed so far. * @@ -52,8 +91,8 @@ export async function CubeSelection( // Clear terminal and move cursor to top process.stdout.write('\x1B[2J\x1B[0f'); - const terminalHeight = process.stdout.rows || 24; - const pageSize = Math.max(10, terminalHeight - 5); + const size = terminalSize(); + const pageSize = Math.max(10, size.rows - 5); console.log('\n Cube Selection\n'); console.log(' Type to filter • Space to select • Enter to confirm\n'); @@ -65,14 +104,15 @@ export async function CubeSelection( multiple: true, choices: cubeChoices, suggest: suggestCubes, + ...size, }); - try { - return { selectedCubes: await prompt.run() }; - } catch { - // User cancelled - return { selectedCubes: [] }; - } + // Deliberately no catch. Swallowing a cancellation here used to return an + // empty selection, which is indistinguishable from "the user picked nothing" + // and let the run carry on to deploy zero cubes. Both ways out now travel: + // a cancellation to `isCancellation` at the CLI boundary, anything else as + // the failure it is. + return { selectedCubes: await prompt.run() }; } export async function AuthSelection(useAuthKey?: boolean): Promise<{ @@ -239,17 +279,17 @@ export async function VariableAssignment( name: 'variables', message: `[${cube.id}] ${cube.name}\n (↑↓ navigate, Enter to submit)`, choices, + ...terminalSize(), }); - try { - const result = await form.run(); - const coercedResult: Record = {}; - for (const [key, value] of Object.entries(result)) { - const zodType = schema[key]; - coercedResult[key] = zodType ? coerceValue(value, zodType) : value; - } - variables.assign(cube.id, 'prompt', coercedResult); - } catch { - // User cancelled + // Deliberately no catch — see `CubeSelection`. A cancelled form used to be + // swallowed here, leaving the cube short of values only the user could give + // and the run continuing as though the form had succeeded. + const result = await form.run(); + const coercedResult: Record = {}; + for (const [key, value] of Object.entries(result)) { + const zodType = schema[key]; + coercedResult[key] = zodType ? coerceValue(value, zodType) : value; } + variables.assign(cube.id, 'prompt', coercedResult); } diff --git a/packages/nopy/src/nopy.session.ts b/packages/nopy/src/nopy.session.ts index 2d1644a..65f1a99 100644 --- a/packages/nopy/src/nopy.session.ts +++ b/packages/nopy/src/nopy.session.ts @@ -6,6 +6,7 @@ import fs from 'node:fs'; import path from 'node:path'; import type { TVariables } from './nopy.common.js'; +import { NopyUsageError } from './nopy.errors.js'; /** * Primitive value types that can be stored in session variables @@ -31,7 +32,13 @@ export interface CubeSession { * Authentication configuration for a session */ export interface AuthSession { - /** Authentication method */ + /** + * Authentication method. + * + * `ssh` is not a third kind of credential — it means the connector owns + * authentication and nopy supplies none. It is what an `@vagrant/` or + * `@docker/` host gets, and nothing prompts for it. + */ method: 'ssh-key' | 'password' | 'ssh'; /** Username for authentication (password auth only) */ username?: string; @@ -40,8 +47,20 @@ export interface AuthSession { /** * Complete session configuration + * + * Everything but `cubes` and `auth` is optional, because a hand-written session + * is a first-class one — the loader requires exactly what it cannot work without. + * `version`, `timestamp` and `name` are stamped on every session nopy writes and + * never demanded of one it reads. */ export interface NopySession { + /** + * Format version of the file. Absent on every session written before this was + * stamped, and on most hand-written ones. + */ + version?: string; + /** ISO 8601 time the session was created */ + timestamp?: string; /** Optional session name */ name?: string; /** Array of cube configurations */ @@ -54,6 +73,40 @@ export interface NopySession { env?: TVariables; } +/** + * The format version stamped into every session nopy writes. + * + * There is one, and nothing yet reads it to decide anything — it exists so that + * a future change to the shape can tell an old file from a new one, which is + * impossible after the fact. + */ +export const SESSION_VERSION = '1.0.0'; + +/** + * A one-line description of a session: `YYYY-MM-DD HH:mm - cubes → hosts`. + * + * Shared with the history list, which is where the format comes from — the two + * name the same thing and there is no reason for them to disagree. + */ +export function describeSession(session: NopySession, timestamp: string): string { + const dateStr = new Date(timestamp).toLocaleString('en-US', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hour12: false, + }); + + const cubeNames = session.cubes.map((c) => c.key).join(', '); + const truncatedCubes = cubeNames.length > 40 ? `${cubeNames.substring(0, 37)}...` : cubeNames; + + const hosts = session.hosts?.join(', ') || 'no host'; + const truncatedHosts = hosts.length > 20 ? `${hosts.substring(0, 17)}...` : hosts; + + return `${dateStr} - ${truncatedCubes} → ${truncatedHosts}`; +} + /** * Saves a session to a JSON file * @@ -129,7 +182,7 @@ function loadSessionFromJSON(filePath: string): NopySession { */ export async function loadSession(filePath: string): Promise { if (!fs.existsSync(filePath)) { - throw new Error(`Session file not found: ${filePath}`); + throw new NopyUsageError(`Session file not found: ${filePath}`); } const ext = path.extname(filePath); @@ -140,23 +193,44 @@ export async function loadSession(filePath: string): Promise { } else if (ext === '.json') { session = loadSessionFromJSON(filePath); } else { - throw new Error(`Unsupported session file format: ${ext}. Use .json or .mjs`); + throw new NopyUsageError(`Unsupported session file format: ${ext}. Use .json or .mjs`); } // Validate required fields if (!session.cubes || !Array.isArray(session.cubes)) { - throw new Error('Invalid session format: missing or invalid "cubes" field'); + throw new NopyUsageError('Invalid session format: missing or invalid "cubes" field'); } if (session.hosts && !Array.isArray(session.hosts)) { - throw new Error('Invalid session format: invalid "hosts" field'); + throw new NopyUsageError('Invalid session format: invalid "hosts" field'); } if (!session.auth) { - throw new Error('Invalid session format: missing "auth" field'); + throw new NopyUsageError('Invalid session format: missing "auth" field'); + } + + // A version this build does not know is a warning, never a refusal: the file + // may well still load, and a session is often the only record of a deployment. + // A missing version says nothing at all — it predates the stamp. + if (session.version !== undefined && session.version !== SESSION_VERSION) { + console.error( + `Warning: session "${filePath}" declares version ${session.version}; ` + + `this build writes ${SESSION_VERSION}. Loading it anyway.` + ); } return session; } +/** + * Suffixes {@link listSessions} recognises. + * + * `.nopysession.*` is the documented name and the one the README's examples use; + * it was not matched at all, because `wild.nopysession.json` does not end in + * `.session.json` — the dot before `session` is part of the suffix. The shorter + * pair stays recognised: `saveSession` writes whatever path it is given, so + * files under the old name exist and there is no reason to stop finding them. + */ +const SESSION_SUFFIXES = ['.nopysession.json', '.nopysession.mjs', '.session.json', '.session.mjs']; + /** * Lists all session files in a directory * @@ -170,7 +244,7 @@ export function listSessions(dirPath: string = process.cwd()): string[] { const files = fs.readdirSync(dirPath); return files - .filter((file) => file.endsWith('.session.json') || file.endsWith('.session.mjs')) + .filter((file) => SESSION_SUFFIXES.some((suffix) => file.endsWith(suffix))) .map((file) => path.join(dirPath, file)); } @@ -186,8 +260,12 @@ export function createSession(params: { hosts: string[]; auth: AuthSession; env?: TVariables; + /** Overrides the creation time; for tests, and for re-stamping a replay. */ + timestamp?: string; }): NopySession { return { + version: SESSION_VERSION, + timestamp: params.timestamp ?? new Date().toISOString(), name: params.name, cubes: params.cubes, hosts: params.hosts, diff --git a/packages/nopy/src/nopy.workflow.ts b/packages/nopy/src/nopy.workflow.ts index da82923..19aa7b1 100644 --- a/packages/nopy/src/nopy.workflow.ts +++ b/packages/nopy/src/nopy.workflow.ts @@ -35,8 +35,18 @@ export interface WorkflowResult { username?: string; /** Password if applicable */ password?: string; - /** Whether this is a session replay */ - isReplay: boolean; + /** + * Where a replayed session came from, or `undefined` for a fresh interactive + * run. + * + * Was a boolean, which conflated two runs that need different treatment: a + * `-R`/`-H` replay is already in history and must not be recorded again, while + * a `--load-session` run is not in history at all — recording it is the only + * way `nopy history` and `-R` can see it afterwards. Everything that merely + * asks "am I replaying?" (reading values back off the session rather than + * prompting) takes `replaySource !== undefined`. + */ + replaySource?: 'file' | 'history'; } /** @@ -83,7 +93,7 @@ export async function runInteractiveWorkflow( authMethod: authResult.authMethod, username: authResult.username, password: authResult.password, - isReplay: false, + replaySource: undefined, }; } @@ -141,7 +151,7 @@ export async function runReplayWorkflow( authMethod, username, password, - isReplay: true, + replaySource: 'file', }; } @@ -196,7 +206,7 @@ export async function runSessionReplayWorkflow( authMethod, username, password, - isReplay: true, + replaySource: 'history', }; } diff --git a/packages/nopy/tests/common.test.ts b/packages/nopy/tests/common.test.ts index 9086983..76f58a3 100644 --- a/packages/nopy/tests/common.test.ts +++ b/packages/nopy/tests/common.test.ts @@ -200,4 +200,67 @@ describe('Variables secrets', () => { expect(variables.persistable('cube-a')).toEqual({}); expect(variables.persistable('cube-b')).toEqual({ PASSWORD: 'b' }); }); + + it('excludes a declared secret from the env a session records', () => { + const variables = new Variables({ PASSWORD: 'hunter2', KEY_DIR: './vault' }, ['PASSWORD']); + + expect(variables.persistableEnv()).toEqual({ KEY_DIR: './vault' }); + }); + + it('records an env with no secrets in it whole', () => { + const variables = new Variables({ KEY_DIR: './vault' }, ['PASSWORD']); + + expect(variables.persistableEnv()).toEqual({ KEY_DIR: './vault' }); + }); +}); + +describe('Variables globally declared secrets', () => { + /** `env` carrying a key that cube-a declares secret and cube-b knows nothing of. */ + const withLeakyEnv = () => { + const variables = new Variables({ PASSWORD: 'wildpass123', KEY_DIR: '/vault' }, ['PASSWORD']); + variables.declareSecrets('cube-a', ['PASSWORD']); + variables.declareSchema('cube-a', ['USER', 'PASSWORD']); + variables.declareSchema('cube-b', ['PORT']); + return variables; + }; + + it('does not seed a secret onto a cube that does not declare it', () => { + const variables = withLeakyEnv(); + variables.assign('cube-b', 'default', { PORT: 22 }); + + expect(variables.get('cube-b')).not.toHaveProperty('PASSWORD'); + expect(variables.of('cube-b', 'PASSWORD')).toBeUndefined(); + }); + + it('still seeds it onto a cube whose schema declares it', () => { + const variables = withLeakyEnv(); + variables.assign('cube-a', 'default', {}); + + expect(variables.get('cube-a').PASSWORD).toBe('wildpass123'); + expect(variables.of('cube-a', 'PASSWORD')?.origin).toBe('env'); + expect(variables.persistable('cube-a')).not.toHaveProperty('PASSWORD'); + }); + + it('keeps broadcasting an undeclared key that is not a secret', () => { + // ssh:keyman reads KEY_DIR off host.data without declaring it in its schema. + const variables = withLeakyEnv(); + variables.assign('cube-b', 'default', {}); + + expect(variables.get('cube-b').KEY_DIR).toBe('/vault'); + }); + + it('redacts a global secret on a cube whose own manifest forgot to list it', () => { + const variables = new Variables({}, ['PASSWORD']); + variables.assign('cube-b', 'prompt', { PASSWORD: 'typed' }); + + expect(variables.of('cube-b', 'PASSWORD')?.redacted).toBe(true); + expect(variables.persistable('cube-b')).toEqual({}); + }); + + it('treats a cube that declared no schema as declaring nothing', () => { + const variables = new Variables({ PASSWORD: 'p' }, ['PASSWORD']); + variables.assign('cube-z', 'default', {}); + + expect(variables.get('cube-z')).toEqual({}); + }); }); diff --git a/packages/nopy/tests/cubes.dependencies.edge.test.ts b/packages/nopy/tests/cubes.dependencies.edge.test.ts index 1db4f95..a2781dd 100644 --- a/packages/nopy/tests/cubes.dependencies.edge.test.ts +++ b/packages/nopy/tests/cubes.dependencies.edge.test.ts @@ -129,10 +129,15 @@ describe('BuildContext session replay', () => { }); describe('BuildContext replay gaps', () => { - const replay = (cube: Cube, recorded: Record = {}, options = {}) => + const replay = ( + cube: Cube, + recorded: Record = {}, + options = {}, + variables = new Variables() + ) => new BuildContext( { [cube.id]: cube }, - new Variables(), + variables, session([{ key: cube.id, variables: recorded }]), config, { method: 'ssh' }, @@ -175,16 +180,18 @@ describe('BuildContext replay gaps', () => { expect(VariableAssignment).not.toHaveBeenCalled(); }); - it('refuses to deploy when the form was cancelled', async () => { + it('refuses to deploy when the form came back empty', async () => { const cube = testCube('cube-a', z.object({ SSID: z.string() })); - // The real VariableAssignment swallows a cancelled form, so the gap check - // has to run again afterwards or the cube ships without the variable. + // A form that resolves is not proof of an answer: enquirer renders + // `Math.min(limit, height)` fields, so a terminal misreporting its height + // submits `{}` without the user having seen a question. The gap check has + // to run again afterwards or the cube ships without the variable. vi.mocked(VariableAssignment).mockResolvedValue(undefined); const context = replay(cube); await expect(context.resolveCube('cube-a', 'host1')).rejects.toThrow( - 'Cube "cube-a" is missing SSID and cannot be deployed.' + 'Cube "cube-a" is missing SSID. Nothing supplied it' ); expect(context.deployCalls).toHaveLength(0); }); @@ -194,10 +201,36 @@ describe('BuildContext replay gaps', () => { const context = replay(cube, {}, { useDefaults: true }); + // A schema default is deliberately not good enough for a secret: it would + // deploy a different credential than the run being replayed. await expect(context.resolveCube('cube-a', 'host1')).rejects.toThrow( - /cannot be replayed with --use-defaults: PASSWORD/ + /cannot be replayed with --use-defaults: PASSWORD would have to be entered\..*not accepted for a secret/s ); }); + + it('accepts a secret supplied through config env under --use-defaults', async () => { + const cube = secretCube('cube-a', z.object({ PASSWORD: z.string().default('changeme') })); + const context = replay( + cube, + {}, + { useDefaults: true }, + new Variables({ PASSWORD: 'from-env' }, ['PASSWORD']) + ); + + await context.resolveCube('cube-a', 'host1'); + + expect(VariableAssignment).not.toHaveBeenCalled(); + expect(context.deployCalls[0].env.PASSWORD).toBe('from-env'); + }); + + it('accepts a required variable a dependency passed under --use-defaults', async () => { + const cube = testCube('cube-a', z.object({ SSID: z.string() })); + const context = replay(cube, {}, { useDefaults: true }); + + await context.resolveCube('cube-a', 'host1', { SSID: 'from-param' }); + + expect(context.deployCalls[0].env.SSID).toBe('from-param'); + }); }); describe('BuildContext session recording', () => { @@ -240,6 +273,50 @@ describe('BuildContext session recording', () => { }); }); +describe('BuildContext secret broadcast', () => { + // The field run put PASSWORD under `env` because the docs said to, and watched + // it appear unmasked on the command line of every cube that was not user:add. + const resolveBoth = async () => { + const declaring = secretCube('cube-a', z.object({ PASSWORD: z.string().default('changeme') })); + const innocent = testCube('cube-b', z.object({ PORT: z.string().default('22') })); + const context = new BuildContext( + { 'cube-a': declaring, 'cube-b': innocent }, + new Variables({ PASSWORD: 'wildpass123', KEY_DIR: '/vault' }, ['PASSWORD']), + session(), + config, + { method: 'ssh' }, + { useDefaults: true } + ); + + await context.resolveCube('cube-a', 'host1'); + await context.resolveCube('cube-b', 'host1'); + return context; + }; + + it('never puts an env secret on a cube that does not declare it', async () => { + const context = await resolveBoth(); + const [, forB] = context.deployCalls; + + expect(forB.cube).toBe('cube-b'); + expect(forB.env).not.toHaveProperty('PASSWORD'); + expect(forB.command.join(' ')).not.toContain('wildpass123'); + }); + + it('still delivers it to the cube that declares it', async () => { + const context = await resolveBoth(); + const [forA] = context.deployCalls; + + expect(forA.env.PASSWORD).toBe('wildpass123'); + expect(forA.secrets).toEqual(['PASSWORD']); + }); + + it('leaves an ordinary env key broadcast to both', async () => { + const context = await resolveBoth(); + + expect(context.deployCalls.map((call) => call.env.KEY_DIR)).toEqual(['/vault', '/vault']); + }); +}); + describe('BuildContext --use-defaults', () => { const withDefaults = (cube: Cube, variables = new Variables(), cfg = config) => new BuildContext( @@ -333,6 +410,38 @@ describe('BuildContext --use-defaults', () => { }); }); +describe('BuildContext interactive completeness', () => { + const interactive = (cube: Cube, variables = new Variables()) => + new BuildContext({ [cube.id]: cube }, variables, session(), config, { method: 'ssh' }); + + it('refuses to deploy when the form submitted nothing', async () => { + const cube = testCube('cube-a', z.object({ SSID: z.string() })); + // What a 0-row terminal does: the form renders no fields, the user sees no + // question, enquirer resolves `{}` and the run used to carry on and deploy + // the cube with SSID simply absent from `--data`. + vi.mocked(VariableAssignment).mockResolvedValue(undefined); + + const context = interactive(cube); + + await expect(context.resolveCube('cube-a', 'host1')).rejects.toThrow( + /Cube "cube-a" is missing SSID\. Nothing supplied it/ + ); + expect(context.deployCalls).toHaveLength(0); + }); + + it('deploys when the form answered', async () => { + const cube = testCube('cube-a', z.object({ SSID: z.string() })); + vi.mocked(VariableAssignment).mockImplementation(async (_cube, variables) => { + variables.assign('cube-a', 'prompt', { SSID: 'typed' }); + }); + + const context = interactive(cube); + await context.resolveCube('cube-a', 'host1'); + + expect(context.deployCalls[0].env.SSID).toBe('typed'); + }); +}); + describe('BuildContext command construction', () => { const build = (auth: { method: string; username?: string; password?: string }) => { const context = new BuildContext( diff --git a/packages/nopy/tests/cubes.dependencies.test.ts b/packages/nopy/tests/cubes.dependencies.test.ts index ecf5ee9..6ff3ad8 100644 --- a/packages/nopy/tests/cubes.dependencies.test.ts +++ b/packages/nopy/tests/cubes.dependencies.test.ts @@ -125,3 +125,53 @@ describe('BuildContext.resolveCube', () => { expect(context.deployCalls.map((c) => c.cube)).toEqual(['cube-a', 'cube-b', 'cube-c']); }); }); + +describe('deploy order across several selected cubes', () => { + // `nopy.main.ts` walks `workflow.selectedCubes` and calls `resolveCube` once + // per entry, so the order that list arrives in is the order the loop visits. + // Emission is post-order, though, so a declared edge is honoured whichever way + // round the two cubes were listed — the recursion *is* the topological sort, + // and these pin that rather than leaving it to be inferred from the one-root + // cases above. + async function resolveAll(cubes: Record, selected: string[]): Promise { + const context = new BuildContext( + cubes, + new Variables(), + { cubes: [] } as any, + { + env: {}, + } as any, + { method: 'ssh' } + ); + + for (const id of selected) await context.resolveCube(id, 'host1'); + + return context.deployCalls.map((c) => c.cube); + } + + it('emits a dependency first even when it is selected last', async () => { + const cubes = { + 'cube-a': createTestCube('cube-a'), + 'cube-b': createTestCube('cube-b', () => ['cube-a']), + }; + + // The list order is the inversion of the dependency: b depends on a, and a + // is named after it. Resolving b still drags a in ahead of itself, and the + // second visit is deduped rather than re-emitted at the tail. + expect(await resolveAll(cubes, ['cube-b', 'cube-a'])).toEqual(['cube-a', 'cube-b']); + expect(await resolveAll(cubes, ['cube-a', 'cube-b'])).toEqual(['cube-a', 'cube-b']); + }); + + it('interleaves an unrelated cube by list order and nothing else', async () => { + // With no edge between them there is nothing to sort on, so `cube-z` lands + // where the list put it. That is the whole of what selection order decides. + const cubes = { + 'cube-a': createTestCube('cube-a'), + 'cube-b': createTestCube('cube-b', () => ['cube-a']), + 'cube-z': createTestCube('cube-z'), + }; + + expect(await resolveAll(cubes, ['cube-z', 'cube-b'])).toEqual(['cube-z', 'cube-a', 'cube-b']); + expect(await resolveAll(cubes, ['cube-b', 'cube-z'])).toEqual(['cube-a', 'cube-b', 'cube-z']); + }); +}); diff --git a/packages/nopy/tests/errors.test.ts b/packages/nopy/tests/errors.test.ts new file mode 100644 index 0000000..7f1518f --- /dev/null +++ b/packages/nopy/tests/errors.test.ts @@ -0,0 +1,62 @@ +/** + * Tests for nopy.errors — how a failed run is presented. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NopyUsageError, reportError } from '../src/nopy.errors.js'; + +let out: ReturnType; +let err: ReturnType; + +/** Everything written to stderr by the last call, as one string. */ +const stderr = () => err.mock.calls.map((call) => String(call[0])).join('\n'); + +beforeEach(() => { + out = vi.spyOn(console, 'log').mockImplementation(() => {}); + err = vi.spyOn(console, 'error').mockImplementation(() => {}); + delete process.env.NOPY_DEBUG; +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('reportError', () => { + it('prints a usage error as one line and points at the debug switch', () => { + reportError(new NopyUsageError('No .nopyrc.json found')); + + expect(stderr()).toContain('Error: No .nopyrc.json found'); + expect(stderr()).not.toContain('nopy.errors'); + expect(stderr()).toContain('NOPY_DEBUG=1'); + }); + + it('keeps the stack for anything unexpected', () => { + reportError(new TypeError('cannot read properties of undefined')); + + expect(stderr()).toContain('Error: cannot read properties of undefined'); + expect(stderr()).toContain('TypeError: cannot read properties of undefined\n at '); + expect(stderr()).not.toContain('NOPY_DEBUG=1'); + }); + + it('prints the stack of a usage error under NOPY_DEBUG', () => { + process.env.NOPY_DEBUG = '1'; + + reportError(new NopyUsageError('No .nopyrc.json found')); + + expect(stderr()).toContain('NopyUsageError: No .nopyrc.json found\n at '); + expect(stderr()).not.toContain('NOPY_DEBUG=1 for'); + }); + + it('reports a thrown non-error', () => { + reportError('just a string'); + + expect(stderr()).toContain('Error: just a string'); + expect(stderr()).toContain('NOPY_DEBUG=1'); + }); + + it('says nothing on stdout', () => { + reportError(new NopyUsageError('No .nopyrc.json found')); + + expect(out).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/nopy/tests/executor.test.ts b/packages/nopy/tests/executor.test.ts index a6ea456..2941741 100644 --- a/packages/nopy/tests/executor.test.ts +++ b/packages/nopy/tests/executor.test.ts @@ -123,20 +123,6 @@ describe('outputExecutionPlan', () => { expect(output).toContain('host1'); }); - it('outputs JSON format when requested', () => { - const calls = [createTestCall('cube-a', 'host1')]; - - outputExecutionPlan(calls, true); - - expect(consoleLogSpy).toHaveBeenCalledTimes(1); - const output = consoleLogSpy.mock.calls[0][0]; - const parsed = JSON.parse(output); - - expect(parsed.plan).toHaveLength(1); - expect(parsed.plan[0].cube).toBe('cube-a'); - expect(parsed.plan[0].host).toBe('host1'); - }); - it('masks variables the manifest declared secret', () => { const call: DeployCall = { ...createTestCall('cube-a', 'host1'), diff --git a/packages/nopy/tests/fixtures/form-probe.ts b/packages/nopy/tests/fixtures/form-probe.ts new file mode 100644 index 0000000..cbbc018 --- /dev/null +++ b/packages/nopy/tests/fixtures/form-probe.ts @@ -0,0 +1,42 @@ +/** + * Runs one real enquirer variable form and prints what it produced. + * + * Driven by `tests/prompts.pty.test.ts` under a pty of a chosen size. Nothing + * here is mocked — the point is the prompt library's own behaviour on a + * terminal that reports no size, which cannot be observed from inside a vitest + * worker because there is no TTY there to misreport. + * + * Prints one line, `NOPY_PROBE `, holding the values the form assigned at + * the `prompt` origin. An empty object means the form submitted nothing. + */ + +import { Cube, Manifest } from '@bitsquare/nopy-cubes'; +import { z } from 'zod'; +import { Variables } from '../../src/nopy.common.js'; +import { VariableAssignment } from '../../src/nopy.prompts.js'; + +const KEYS = ['ALPHA', 'BETA']; + +const cube = new Cube( + Manifest({ + id: 'probe', + name: 'Zero-rows probe', + schema: z.object({ + ALPHA: z.string().describe('First value').default(''), + BETA: z.string().describe('Second value').default(''), + }), + }), + '/cubes/probe', + 'deploy.py' +); + +const variables = new Variables(); +await VariableAssignment(cube, variables); + +const assigned: Record = {}; +for (const key of KEYS) { + const variable = variables.of('probe', key); + if (variable?.origin === 'prompt') assigned[key] = variable.value; +} + +process.stdout.write(`\nNOPY_PROBE ${JSON.stringify(assigned)}\n`); diff --git a/packages/nopy/tests/main.test.ts b/packages/nopy/tests/main.test.ts index 60b09b6..12e0960 100644 --- a/packages/nopy/tests/main.test.ts +++ b/packages/nopy/tests/main.test.ts @@ -48,7 +48,12 @@ vi.mock('../src/cubes/index.js', () => ({ loadCubes })); vi.mock('../src/nopy.config.js', () => ({ loadConfig, getConfigPaths })); vi.mock('../src/nopy.workflow.js', () => ({ runWorkflow })); vi.mock('../src/nopy.history.js', () => ({ addToHistory, DEFAULT_HISTORY_SIZE: 10 })); -vi.mock('../src/nopy.session.js', () => ({ saveSession })); +// Only the writer is a spy — `describeSession` and the version constant are pure +// and the assertions below are about what nopy() actually stamps. +vi.mock('../src/nopy.session.js', async (importOriginal) => ({ + ...(await importOriginal()), + saveSession, +})); vi.mock('../src/cubes/dependencies.js', () => ({ BuildContext: class { resolveCube = resolveCube; @@ -67,16 +72,17 @@ vi.mock('../src/nopy.executor.js', async (importOriginal) => { import { nopy } from '../src/nopy.main.js'; -const session = (): NopySession => - ({ - version: '1.0', - name: 'test', - createdAt: '2026-01-01T00:00:00.000Z', - cubes: [], - hosts: ['web-1'], - auth: { method: 'ssh-key' }, - env: {}, - }) as NopySession; +/** + * A session as bare as the loader will accept one — no `version`, `timestamp` + * or `name`, which is exactly what a hand-written file looks like and what + * `nopy()` has to fill in. + */ +const session = (): NopySession => ({ + cubes: [], + hosts: ['web-1'], + auth: { method: 'ssh-key' }, + env: {}, +}); const call = (cube: string): DeployCall => ({ cube, @@ -88,10 +94,12 @@ const call = (cube: string): DeployCall => ({ }); let logSpy: ReturnType; +let errSpy: ReturnType; beforeEach(() => { vi.clearAllMocks(); logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); state.config = { hosts: ['web-1'], cubeDirs: [], cubePackages: [], env: {} }; state.loadResult = { cubes: { 'cube-a': {} }, errors: [] }; @@ -102,14 +110,19 @@ beforeEach(() => { session: session(), selectedCubes: ['cube-a'], authMethod: 'ssh-key', - isReplay: false, + replaySource: undefined, }); executeDeployCalls.mockResolvedValue([ { cube: 'cube-a', host: 'web-1', success: true, duration: 10 }, ]); }); -const output = () => logSpy.mock.calls.map((c) => c.join(' ')).join('\n'); +/** + * The two streams, kept apart on purpose: stdout carries the deploy commands + * and pyinfra's own output, everything nopy says about itself goes to stderr. + */ +const stdout = () => logSpy.mock.calls.map((c) => c.join(' ')).join('\n'); +const stderr = () => errSpy.mock.calls.map((c) => c.join(' ')).join('\n'); describe('nopy', () => { it('runs the happy path and reports success', async () => { @@ -141,7 +154,7 @@ describe('nopy', () => { session: { ...session(), hosts: ['web-1', 'web-2'] }, selectedCubes: ['cube-a', 'cube-b'], authMethod: 'ssh-key', - isReplay: false, + replaySource: undefined, }); await nopy(); @@ -159,13 +172,13 @@ describe('nopy', () => { expect(runWorkflow).not.toHaveBeenCalled(); }); - it('emits the errors as JSON when jsonOutput is set', async () => { + it('reports them on stderr', async () => { state.loadResult = { cubes: {}, errors: ['bad manifest'] }; - await nopy({ jsonOutput: true }); + await nopy(); - const payload = JSON.parse(logSpy.mock.calls.at(-1)?.[0] as string); - expect(payload).toEqual({ success: false, errors: ['bad manifest'] }); + expect(stderr()).toContain('bad manifest'); + expect(stdout()).toBe(''); }); }); @@ -180,7 +193,7 @@ describe('nopy', () => { await nopy({ continueOnError: true }); - const text = output(); + const text = stderr(); expect(text).toContain('Configuration'); expect(text).toContain('Hosts:'); expect(text).toContain('Cube dirs:'); @@ -198,7 +211,7 @@ describe('nopy', () => { await nopy(); - const text = output(); + const text = stderr(); expect(text).toContain('Configuration'); expect(text).not.toContain('Hosts:'); expect(text).not.toContain('Cube dirs:'); @@ -215,25 +228,20 @@ describe('nopy', () => { await nopy(); - const text = output(); + const text = stderr(); expect(text).toContain('~/.nopyrc.json'); expect(text).toContain('./.nopyrc.json'); expect(text).toContain('/etc/nopy/.nopyrc.json'); }); - it('is suppressed for JSON output', async () => { - await nopy({ jsonOutput: true }); - expect(output()).not.toContain('Configuration'); - }); - it('is suppressed when replaying a session object', async () => { await nopy({ replaySession: session() }); - expect(output()).not.toContain('Configuration'); + expect(stderr()).not.toContain('Configuration'); }); it('is suppressed when replaying a session file', async () => { await nopy({ loadSession: '/tmp/s.json' }); - expect(output()).not.toContain('Configuration'); + expect(stderr()).not.toContain('Configuration'); }); }); @@ -247,17 +255,57 @@ describe('nopy', () => { expect(written.cubes).toEqual(state.cubeSessions); }); - it('does not save a replayed session back to file', async () => { + it('leaves a declared secret out of the recorded env', async () => { + // The session's `env` is a copy of the config's, and used to be copied + // verbatim — writing to disk, in plaintext, the credential that was kept + // out of every cube's `variables` one key below. + state.config = { + ...state.config, + env: { PASSWORD: 'hunter2', KEY_DIR: './vault' }, + secrets: ['PASSWORD'], + }; + + await nopy({ saveSession: '/tmp/out.json' }); + + expect(saveSession.mock.calls[0][0].env).toEqual({ KEY_DIR: './vault' }); + }); + + it('saves a replayed session too', async () => { runWorkflow.mockResolvedValue({ session: session(), selectedCubes: ['cube-a'], authMethod: 'ssh-key', - isReplay: true, + replaySource: 'history', }); await nopy({ saveSession: '/tmp/out.json' }); - expect(saveSession).not.toHaveBeenCalled(); + expect(saveSession).toHaveBeenCalledTimes(1); + expect(saveSession.mock.calls[0][1]).toBe('/tmp/out.json'); + }); + + it('stamps version, timestamp and a derived name', async () => { + await nopy({ saveSession: '/tmp/out.json' }); + + const [written] = saveSession.mock.calls[0]; + expect(written.version).toBe('1.0.0'); + expect(written.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(written.name).toContain('cube-a'); + expect(written.name).toContain('web-1'); + }); + + it('keeps the name and version a replayed session already carried', async () => { + runWorkflow.mockResolvedValue({ + session: { ...session(), version: '0.9.0', name: 'hand-written', timestamp: 'then' }, + selectedCubes: ['cube-a'], + authMethod: 'ssh-key', + replaySource: 'file', + }); + + await nopy({ saveSession: '/tmp/out.json' }); + + const [written] = saveSession.mock.calls[0]; + expect(written).toMatchObject({ version: '0.9.0', name: 'hand-written', timestamp: 'then' }); }); it('does not save when no path is given', async () => { @@ -300,12 +348,12 @@ describe('nopy', () => { expect(addToHistory).not.toHaveBeenCalled(); }); - it('skips history for a replay', async () => { + it('skips history for a replay out of history', async () => { runWorkflow.mockResolvedValue({ session: session(), selectedCubes: ['cube-a'], authMethod: 'ssh-key', - isReplay: true, + replaySource: 'history', }); await nopy(); @@ -313,6 +361,19 @@ describe('nopy', () => { expect(addToHistory).not.toHaveBeenCalled(); }); + it('records a replay out of a session file', async () => { + runWorkflow.mockResolvedValue({ + session: session(), + selectedCubes: ['cube-a'], + authMethod: 'ssh-key', + replaySource: 'file', + }); + + await nopy(); + + expect(addToHistory).toHaveBeenCalledTimes(1); + }); + it('skips history when nothing would be deployed', async () => { state.deployCalls = []; @@ -320,19 +381,36 @@ describe('nopy', () => { expect(addToHistory).not.toHaveBeenCalled(); }); + + it('skips history for a print-only run', async () => { + // Same rule as `--dry-run`: nothing was deployed, so nothing belongs at + // the head of the list `-R` repeats. + await nopy({ printOnly: true }); + + expect(addToHistory).not.toHaveBeenCalled(); + }); }); describe('printOnly', () => { it('prints commands and never executes', async () => { await nopy({ printOnly: true }); - const text = output(); + const text = stdout(); expect(text).toContain('Deploy Commands'); expect(text).toContain('# cube-a -> web-1'); expect(text).toContain('pyinfra web-1 -y cube-a.deploy.py'); expect(executeDeployCalls).not.toHaveBeenCalled(); }); + it('keeps stdout to the commands and nothing else', async () => { + // The whole point of the split: `nopy -P > plan.txt` has to be the plan. + // The config banner and every log line are on the other stream. + await nopy({ printOnly: true }); + + expect(stdout()).not.toContain('Configuration'); + expect(stderr()).toContain('Configuration'); + }); + it('reports the command count as the summary total', async () => { const result = await nopy({ printOnly: true }); @@ -359,17 +437,9 @@ describe('nopy', () => { const [, options] = executeDeployCalls.mock.calls[0]; options.onProgress({ cube: 'cube-a', host: 'web-1', success: true }, 1, 1); options.onProgress({ cube: 'cube-b', host: 'web-1', success: false }, 1, 1); - // Exercises both the ✓ and ✗ branches; logtape writes via console.log. - expect(logSpy).toHaveBeenCalled(); - }); - - it('stays silent on progress when jsonOutput is set', async () => { - await nopy({ jsonOutput: true }); - - const [, options] = executeDeployCalls.mock.calls[0]; - const before = logSpy.mock.calls.length; - options.onProgress({ cube: 'cube-a', host: 'web-1', success: true }, 1, 1); - expect(logSpy.mock.calls.length).toBe(before); + // Exercises both the ✓ and ✗ branches; logtape writes via console.error. + expect(stderr()).toContain('cube-a'); + expect(stderr()).toContain('cube-b'); }); }); }); diff --git a/packages/nopy/tests/prompts.pty.test.ts b/packages/nopy/tests/prompts.pty.test.ts new file mode 100644 index 0000000..7ab5a60 --- /dev/null +++ b/packages/nopy/tests/prompts.pty.test.ts @@ -0,0 +1,77 @@ +/** + * The variable form, on a terminal that reports no size. + * + * This is the one case that cannot be tested from inside a vitest worker: there + * is no TTY there for enquirer to misread, so the mocked tests in + * `prompts.test.ts` prove only that `rows` is *passed*, never that passing it + * matters. Here a real pty is opened at 0x0 — `pty.fork()`'s own default, and + * what `script -q` and some CI terminals report — and a real form is answered + * through it. + * + * Measured both ways while writing this: with `terminalSize()` removed from + * `nopy.prompts.ts`, the form never renders and the driver times out with + * nothing on the wire. + * + * Needs `python3` for the pty; skipped, loudly, where there is none. + */ + +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const EXPECT_PY = fileURLToPath(new URL('../../../scripts/expect.py', import.meta.url)); +const TSX = fileURLToPath(new URL('../node_modules/.bin/tsx', import.meta.url)); +const PROBE = fileURLToPath(new URL('./fixtures/form-probe.ts', import.meta.url)); + +const hasPython = spawnSync('python3', ['--version']).status === 0; + +/** Down arrow — how the form moves from one field to the next. */ +const DOWN = '\u001b[B'; + +const STEPS = [ + { expect: 'ALPHA', send: 'alpha-typed', settle: 0.6 }, + { send: DOWN, settle: 0.4 }, + { send: 'beta-typed', settle: 0.4 }, + { send: '\r', settle: 1.2 }, +]; + +describe.skipIf(!hasPython)('variable form over a pty', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'nopy-pty-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + /** Answers the probe form on a pty of the given size; returns what it assigned. */ + const answerForm = (rows: number, cols: number) => { + const stepsPath = path.join(tmpDir, 'steps.json'); + const logPath = path.join(tmpDir, 'session.log'); + fs.writeFileSync(stepsPath, JSON.stringify(STEPS)); + + execFileSync('python3', [EXPECT_PY, stepsPath, '--', TSX, PROBE], { + env: { + ...process.env, + PTY_ROWS: String(rows), + PTY_COLS: String(cols), + EXPECT_TIMEOUT: '60', + EXPECT_LOG: logPath, + }, + stdio: 'pipe', + }); + + const transcript = fs.readFileSync(logPath, 'utf-8').replace(/\r/g, ''); + const line = transcript.split('\n').find((l) => l.startsWith('NOPY_PROBE ')); + return line ? JSON.parse(line.slice('NOPY_PROBE '.length)) : undefined; + }; + + it('collects every field on a terminal reporting 0x0', () => { + expect(answerForm(0, 0)).toEqual({ ALPHA: 'alpha-typed', BETA: 'beta-typed' }); + }, 90_000); +}); diff --git a/packages/nopy/tests/prompts.test.ts b/packages/nopy/tests/prompts.test.ts index 85fc519..4cc87a3 100644 --- a/packages/nopy/tests/prompts.test.ts +++ b/packages/nopy/tests/prompts.test.ts @@ -55,11 +55,27 @@ const question = (name: string) => questions().find((q) => q.name === name); /** Grabs the options the last enquirer AutoComplete prompt was constructed with. */ const autoComplete = () => autoCompleteCtor.mock.calls.at(-1)?.[0] as Record; +/** Grabs the options the last enquirer Form prompt was constructed with. */ +const formOptions = () => formCtor.mock.calls.at(-1)?.[0] as Record; + /** Grabs the choices the last enquirer Form prompt was constructed with. */ -const formChoices = () => { - const options = formCtor.mock.calls.at(-1)?.[0] as { choices: Record[] }; - return options.choices; -}; +const formChoices = () => formOptions().choices as Record[]; + +/** Runs `body` with the terminal reporting the given size, then puts it back. */ +async function withTerminal( + size: { rows: number; columns: number }, + body: () => Promise +): Promise { + const was = { rows: process.stdout.rows, columns: process.stdout.columns }; + Object.defineProperty(process.stdout, 'rows', { value: size.rows, configurable: true }); + Object.defineProperty(process.stdout, 'columns', { value: size.columns, configurable: true }); + try { + await body(); + } finally { + Object.defineProperty(process.stdout, 'rows', { value: was.rows, configurable: true }); + Object.defineProperty(process.stdout, 'columns', { value: was.columns, configurable: true }); + } +} const cube = (id: string, name: string, schema = z.object({})) => new Cube(Manifest({ id, name, schema }), `/cubes/${id}`, 'deploy.py'); @@ -113,24 +129,42 @@ describe('CubeSelection', () => { it('derives page size from the terminal height', async () => { autoCompleteRun.mockResolvedValue([]); - const rows = process.stdout.rows; - Object.defineProperty(process.stdout, 'rows', { value: 40, configurable: true }); - await CubeSelection(cubes); - expect(autoComplete().limit).toBe(35); + await withTerminal({ rows: 40, columns: 200 }, async () => { + await CubeSelection(cubes); + expect(autoComplete().limit).toBe(35); + }); - // Falls back to a floor of 10 on a short (or unknown) terminal. - Object.defineProperty(process.stdout, 'rows', { value: 0, configurable: true }); - await CubeSelection(cubes); - expect(autoComplete().limit).toBe(19); - - Object.defineProperty(process.stdout, 'rows', { value: rows, configurable: true }); + // A terminal reporting nothing is floored, not believed. + await withTerminal({ rows: 0, columns: 0 }, async () => { + await CubeSelection(cubes); + expect(autoComplete().limit).toBe(19); + }); }); - it('selects nothing when the user cancels', async () => { + it('hands the prompt a window size it can render into', async () => { + autoCompleteRun.mockResolvedValue([]); + + // Passing `rows` is what keeps enquirer away from its own `utils.height`, + // which overwrites a good fallback with `getWindowSize()[1]` — zero here. + await withTerminal({ rows: 0, columns: 0 }, async () => { + await CubeSelection(cubes); + expect(autoComplete()).toMatchObject({ rows: 24, columns: 80 }); + }); + + await withTerminal({ rows: 50, columns: 200 }, async () => { + await CubeSelection(cubes); + expect(autoComplete()).toMatchObject({ rows: 50, columns: 200 }); + }); + }); + + it('lets a cancellation travel instead of returning an empty selection', async () => { + // An empty selection is a legitimate answer, so swallowing the rejection + // here made "the user backed out" and "the user picked nothing" the same + // event and let the run continue to deploy zero cubes. autoCompleteRun.mockRejectedValue(new Error('cancelled')); - await expect(CubeSelection(cubes)).resolves.toEqual({ selectedCubes: [] }); + await expect(CubeSelection(cubes)).rejects.toThrow('cancelled'); }); }); @@ -418,13 +452,31 @@ describe('VariableAssignment', () => { expect(variables.get('svc').extra).toBe('kept'); }); - it('assigns nothing when the user cancels the form', async () => { + it('hands the form a window size it can render into', async () => { + const variables = new Variables(); + formRun.mockResolvedValue({}); + + // The form is where a zero height actually costs something: enquirer + // renders `Math.min(limit, height)` fields, so a 0-row terminal shows none + // of them and submits `{}` without the user ever seeing the questions. + await withTerminal({ rows: 0, columns: 0 }, async () => { + await VariableAssignment(cube('svc', 'Service', schema), variables); + expect(formOptions()).toMatchObject({ rows: 24, columns: 80 }); + }); + + await withTerminal({ rows: 50, columns: 200 }, async () => { + await VariableAssignment(cube('svc', 'Service', schema), variables); + expect(formOptions()).toMatchObject({ rows: 50, columns: 200 }); + }); + }); + + it('lets a cancelled form travel rather than assigning nothing', async () => { const variables = new Variables(); formRun.mockRejectedValue(new Error('cancelled')); - await expect( - VariableAssignment(cube('svc', 'Service', schema), variables) - ).resolves.toBeUndefined(); + await expect(VariableAssignment(cube('svc', 'Service', schema), variables)).rejects.toThrow( + 'cancelled' + ); expect(variables.get('svc')).toEqual({}); }); diff --git a/packages/nopy/tests/session.test.ts b/packages/nopy/tests/session.test.ts index 496a5dc..c658a11 100644 --- a/packages/nopy/tests/session.test.ts +++ b/packages/nopy/tests/session.test.ts @@ -5,12 +5,14 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createSession, + describeSession, listSessions, loadSession, type NopySession, + SESSION_VERSION, saveSession, } from '../src/nopy.session.js'; @@ -48,6 +50,62 @@ describe('createSession', () => { expect(session.env).toEqual({ KEY: 'value' }); }); + + it('stamps the format version and a creation time', () => { + const session = createSession({ cubes: [], hosts: ['localhost'], auth: { method: 'ssh' } }); + + expect(session.version).toBe(SESSION_VERSION); + expect(new Date(session.timestamp!).toISOString()).toBe(session.timestamp); + }); + + it('lets the caller supply the timestamp', () => { + const session = createSession({ + cubes: [], + hosts: ['localhost'], + auth: { method: 'ssh' }, + timestamp: '2026-01-01T00:00:00.000Z', + }); + + expect(session.timestamp).toBe('2026-01-01T00:00:00.000Z'); + }); +}); + +describe('describeSession', () => { + const at = '2026-01-01T12:30:00.000Z'; + + it('names the cubes and the hosts', () => { + const name = describeSession( + { + cubes: [{ key: 'apt:essentials', variables: {} }], + hosts: ['web-1'], + auth: { method: 'ssh' }, + }, + at + ); + + expect(name).toContain('apt:essentials'); + expect(name).toContain('web-1'); + }); + + it('says so when there is no host', () => { + const name = describeSession({ cubes: [], auth: { method: 'ssh' } }, at); + + expect(name).toContain('no host'); + }); + + it('truncates a long cube list and a long host list', () => { + const name = describeSession( + { + cubes: Array.from({ length: 10 }, (_, i) => ({ key: `cube-${i}`, variables: {} })), + hosts: Array.from({ length: 10 }, (_, i) => `host-${i}`), + auth: { method: 'ssh' }, + }, + at + ); + + expect(name).toContain('...'); + expect(name.split('→')[1]).toContain('...'); + }); }); describe('saveSession and loadSession', () => { @@ -116,6 +174,51 @@ describe('saveSession and loadSession', () => { await expect(loadSession(sessionPath)).rejects.toThrow('auth'); }); + + // Half of SESSION_FORMAT.md is about the MJS form, and nothing exercised it. + // Each test needs its own filename: `import()` caches by URL, so a second + // module written to the same path would never be read. + it('loads a session from an MJS default export', async () => { + const mjsPath = path.join(tempDir, 'ok.session.mjs'); + fs.writeFileSync( + mjsPath, + 'export default { cubes: [{ key: "apt:essentials", variables: {} }], auth: { method: "ssh" } };' + ); + + await expect(loadSession(mjsPath)).resolves.toMatchObject({ + cubes: [{ key: 'apt:essentials', variables: {} }], + }); + }); + + it('rejects an MJS session with no default export', async () => { + const mjsPath = path.join(tempDir, 'no-default.session.mjs'); + fs.writeFileSync(mjsPath, 'export const session = {};'); + + await expect(loadSession(mjsPath)).rejects.toThrow('must export a default object'); + }); + + it('loads a session with no version at all', async () => { + fs.writeFileSync(sessionPath, JSON.stringify({ cubes: [], auth: { method: 'ssh' } })); + const warn = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(loadSession(sessionPath)).resolves.toMatchObject({ cubes: [] }); + expect(warn).not.toHaveBeenCalled(); + + warn.mockRestore(); + }); + + it('warns about an unknown version but still loads it', async () => { + fs.writeFileSync( + sessionPath, + JSON.stringify({ version: '9.9.9', cubes: [], auth: { method: 'ssh' } }) + ); + const warn = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(loadSession(sessionPath)).resolves.toMatchObject({ version: '9.9.9' }); + expect(warn.mock.calls[0][0]).toContain('9.9.9'); + + warn.mockRestore(); + }); }); describe('listSessions', () => { @@ -154,4 +257,18 @@ describe('listSessions', () => { expect(result).toHaveLength(1); expect(result[0].endsWith('test.session.mjs')).toBe(true); }); + + it('finds the documented .nopysession.* files', () => { + // The name every example in the README uses, and the one this missed: + // `wild.nopysession.json` does not end in `.session.json`. + fs.writeFileSync(path.join(tempDir, 'wild.nopysession.json'), '{}'); + fs.writeFileSync(path.join(tempDir, 'wild.nopysession.mjs'), 'export default {}'); + fs.writeFileSync(path.join(tempDir, 'nopysession.json'), '{}'); + + const result = listSessions(tempDir); + + expect(result).toHaveLength(2); + expect(result.some((p) => p.endsWith('wild.nopysession.json'))).toBe(true); + expect(result.some((p) => p.endsWith('wild.nopysession.mjs'))).toBe(true); + }); }); diff --git a/packages/nopy/tests/workflow.test.ts b/packages/nopy/tests/workflow.test.ts index f224e41..2dc8f1f 100644 --- a/packages/nopy/tests/workflow.test.ts +++ b/packages/nopy/tests/workflow.test.ts @@ -78,7 +78,7 @@ describe('runInteractiveWorkflow', () => { expect(result.selectedCubes).toEqual(['cube-a']); expect(result.authMethod).toBe('ssh-key'); - expect(result.isReplay).toBe(false); + expect(result.replaySource).toBeUndefined(); expect(result.session.hosts).toEqual(['web-1']); expect(result.session.env).toEqual({ GLOBAL: 'value' }); expect(mockHostSelection).toHaveBeenCalledWith(config.hosts); @@ -150,7 +150,7 @@ describe('runReplayWorkflow', () => { const result = await runReplayWorkflow('/tmp/s.json', cubes, config); expect(mockLoadSession).toHaveBeenCalledWith('/tmp/s.json'); - expect(result.isReplay).toBe(true); + expect(result.replaySource).toBe('file'); expect(result.selectedCubes).toEqual(['cube-a']); expect(mockHostSelection).not.toHaveBeenCalled(); expect(mockPasswordSelection).not.toHaveBeenCalled(); @@ -225,7 +225,7 @@ describe('runSessionReplayWorkflow', () => { it('replays an in-memory session without prompting', async () => { const result = await runSessionReplayWorkflow(session(), cubes, config); - expect(result.isReplay).toBe(true); + expect(result.replaySource).toBe('history'); expect(result.selectedCubes).toEqual(['cube-a']); expect(mockLoadSession).not.toHaveBeenCalled(); expect(mockHostSelection).not.toHaveBeenCalled(); @@ -287,7 +287,7 @@ describe('runWorkflow dispatch', () => { it('prefers an in-memory replay session over everything else', async () => { const result = await runWorkflow('/tmp/s.json', cubes, config, {}, session()); - expect(result.isReplay).toBe(true); + expect(result.replaySource).toBe('history'); expect(mockLoadSession).not.toHaveBeenCalled(); expect(mockCubeSelection).not.toHaveBeenCalled(); }); @@ -298,14 +298,14 @@ describe('runWorkflow dispatch', () => { const result = await runWorkflow('/tmp/s.json', cubes, config); expect(mockLoadSession).toHaveBeenCalledWith('/tmp/s.json'); - expect(result.isReplay).toBe(true); + expect(result.replaySource).toBe('file'); expect(mockCubeSelection).not.toHaveBeenCalled(); }); it('falls back to the interactive workflow', async () => { const result = await runWorkflow(undefined, cubes, config, { useAuthKey: true }); - expect(result.isReplay).toBe(false); + expect(result.replaySource).toBeUndefined(); expect(mockCubeSelection).toHaveBeenCalled(); expect(mockAuthSelection).toHaveBeenCalledWith(true); }); diff --git a/scripts/drive.py b/scripts/drive.py new file mode 100755 index 0000000..9520193 --- /dev/null +++ b/scripts/drive.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Drive an interactive CLI through a pty on a fixed schedule. + + scripts/drive.py -- [args...] + +`script.json` is a list of `[seconds_since_start, "text to send"]` pairs. +Everything the child prints is echoed to this process's stdout. + +Timing-based, so it is the blunt one — good for a quick manual poke at the TUI, +bad for anything that has to be reliable. Prefer `expect.py`, which waits for the +prompt instead of guessing when it will appear. + +Environment: `PTY_ROWS` / `PTY_COLS` (default 50x200), `DRIVE_TIMEOUT` seconds. +""" + +import json +import os +import select +import signal +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from ptysize import size_from_env, spawn # noqa: E402 + +sep = sys.argv.index("--") +steps = json.loads(open(sys.argv[1]).read()) +cmd = sys.argv[sep + 1 :] + +rows, cols = size_from_env() +pid, fd = spawn(cmd, rows, cols) + +start = time.time() +pending = list(steps) +deadline = start + float(os.environ.get("DRIVE_TIMEOUT", "3600")) + +while True: + if pending and time.time() - start >= pending[0][0]: + _, text = pending.pop(0) + os.write(fd, text.encode()) + r, _, _ = select.select([fd], [], [], 0.2) + if r: + try: + data = os.read(fd, 65536) + except OSError: + break + if not data: + break + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + if time.time() > deadline: + os.kill(pid, signal.SIGKILL) + break + +_, status = os.waitpid(pid, 0) +sys.stderr.write("\n[drive.py] exit status: %d\n" % (status >> 8)) +sys.exit(status >> 8) diff --git a/scripts/expect.py b/scripts/expect.py new file mode 100755 index 0000000..873c89e --- /dev/null +++ b/scripts/expect.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Expect-style pty driver — waits for each prompt before answering it. + + scripts/expect.py -- [args...] + +`script.json` is a list of steps: + + {"expect": "", "send": "", "settle": 0.4} + {"send": ""} -- send immediately + +Each regex is matched against the ANSI-stripped output accumulated *since the +previous step completed*, not against everything seen so far, so the same prompt +text can be awaited twice in one run — which the nopy variable form does, once +per cube. + +Environment: `PTY_ROWS` / `PTY_COLS` (default 50x200), `EXPECT_TIMEOUT` seconds, +`EXPECT_LOG` for the transcript path (default `expect.log`). +""" + +import json +import os +import re +import select +import signal +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from ptysize import size_from_env, spawn # noqa: E402 + +ANSI = re.compile(rb"\x1b\[[0-9;?]*[a-zA-Z]|\x1b\][^\x07]*\x07|\r") + +sep = sys.argv.index("--") +steps = json.loads(open(sys.argv[1]).read()) +cmd = sys.argv[sep + 1 :] +TIMEOUT = float(os.environ.get("EXPECT_TIMEOUT", "900")) + +rows, cols = size_from_env() +pid, fd = spawn(cmd, rows, cols) + +log = open(os.environ.get("EXPECT_LOG", "expect.log"), "wb") +start = time.time() +window = b"" # output since the last completed step +alive = True + + +def pump(seconds): + """Read child output for `seconds`, appending to `window`.""" + global window, alive + end = time.time() + seconds + while time.time() < end: + r, _, _ = select.select([fd], [], [], min(0.2, max(0.01, end - time.time()))) + if not r: + continue + try: + data = os.read(fd, 65536) + except OSError: + alive = False + return + if not data: + alive = False + return + window += data + log.write(data) + log.flush() + + +failed = False + +for i, step in enumerate(steps): + pattern = step.get("expect") + if pattern: + rx = re.compile(pattern.encode()) + found = False + while time.time() - start < TIMEOUT: + if rx.search(ANSI.sub(b"", window)): + found = True + break + if not alive: + break + pump(0.3) + if not found: + sys.stderr.write( + "\n[expect] step %d timed out waiting for %r\n" % (i, pattern) + ) + failed = True + os.kill(pid, signal.SIGKILL) + break + sys.stderr.write("[expect] step %d matched %r\n" % (i, pattern)) + pump(step.get("settle", 0.5)) + window = b"" + text = step.get("send") + if text: + os.write(fd, text.encode()) + +# drain until the child exits +while alive and time.time() - start < TIMEOUT: + pump(1.0) +if alive: + sys.stderr.write("\n[expect] overall timeout, killing child\n") + failed = True + os.kill(pid, signal.SIGKILL) + +_, status = os.waitpid(pid, 0) +log.close() +code = status >> 8 +sys.stderr.write("\n[expect] exit status: %d\n" % code) +# A driver that gave up must not report the child's exit code as its own — a +# SIGKILLed child can still look like a clean 0 to a caller reading $?. +sys.exit(1 if failed else code) diff --git a/scripts/ptysize.py b/scripts/ptysize.py new file mode 100644 index 0000000..306800b --- /dev/null +++ b/scripts/ptysize.py @@ -0,0 +1,50 @@ +"""Spawn a child on a pty whose window size is what you asked for. + +`pty.fork()` leaves the new terminal at **0 rows by 0 columns**, and nothing +fixes that afterwards: `COLUMNS`/`LINES` in the environment are a shell +convention that `ioctl(TIOCGWINSZ)` has never heard of, so `process.stdout.rows` +in the child stays 0 however they are set. + +That is not a detail. A driver that forgets the ioctl is testing a terminal no +user has, and it lied to us once already — the enquirer `Form` under it returned +`{}` for reasons that had nothing to do with the code under test. See the +`terminalSize` comment in `packages/nopy/src/nopy.prompts.ts`. + +So the size is always set explicitly here, including when it is set to 0: the +degenerate terminal is worth testing, but only on purpose. +""" + +import fcntl +import os +import pty +import struct +import termios + + +def set_winsize(fd, rows, cols): + fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)) + + +def spawn(cmd, rows, cols, env=None): + """Fork `cmd` onto a pty sized `rows` x `cols`. Returns (pid, fd).""" + pid, fd = pty.fork() + if pid == 0: + os.environ["TERM"] = "xterm-256color" + # Kept in step with the ioctl so that a program reading either one gets + # the same answer. The ioctl is what actually matters. + os.environ["COLUMNS"] = str(cols) + os.environ["LINES"] = str(rows) + for key, value in (env or {}).items(): + os.environ[key] = value + os.execvp(cmd[0], cmd) + + set_winsize(fd, rows, cols) + return pid, fd + + +def size_from_env(default_rows=50, default_cols=200): + """`PTY_ROWS` / `PTY_COLS`, so a caller can ask for the 0x0 case.""" + return ( + int(os.environ.get("PTY_ROWS", default_rows)), + int(os.environ.get("PTY_COLS", default_cols)), + )