[refactor] moving cubes into own package"
Publish snapshot / snapshot (push) Successful in 1m2s

[fix] default parameter run records parameters in session for replay[fix] remove default parameters for several cubes
This commit is contained in:
Benjamin Diedrichsen
2026-07-28 12:18:10 +02:00
parent ac050c4459
commit 6ecb2c366f
130 changed files with 3386 additions and 520 deletions
+7
View File
@@ -81,6 +81,13 @@ jobs:
echo "::endgroup::" echo "::endgroup::"
done done
- name: Verify the packed manifests
# `workspace:*` is mandatory in the manifests but meaningless to npm, so
# a range that survives into a tarball is an install failure for every
# consumer. The publish workflows run this too; running it here is what
# puts the failure on the pull request instead of on the release.
run: node scripts/verify-pack.mjs
- name: Upload coverage reports - name: Upload coverage reports
if: always() if: always()
continue-on-error: true continue-on-error: true
+27 -10
View File
@@ -1,5 +1,5 @@
# Every commit that lands on `main` publishes a prerelease of both packages to # Every commit that lands on `main` publishes a prerelease of every publishable
# the Gitea npm registry under the `main` dist-tag: # package to the Gitea npm registry under the `main` dist-tag:
# #
# pnpm add @bitsquare/nopy@main # pnpm add @bitsquare/nopy@main
# #
@@ -78,6 +78,11 @@ jobs:
# Explicit, so the publish step can skip lifecycle scripts entirely. # Explicit, so the publish step can skip lifecycle scripts entirely.
run: pnpm run build run: pnpm run build
- name: Verify the packed manifests
# Packages link to each other with `workspace:*`, which npm cannot
# install. Proves on the tarball that pack rewrote it.
run: node scripts/verify-pack.mjs
- name: Authenticate against the Gitea registry - name: Authenticate against the Gitea registry
run: | run: |
set -euo pipefail set -euo pipefail
@@ -97,23 +102,35 @@ jobs:
export npm_config_userconfig="$NPMRC" export npm_config_userconfig="$NPMRC"
: "${GITHUB_STEP_SUMMARY:=/dev/null}" : "${GITHUB_STEP_SUMMARY:=/dev/null}"
short_sha=$(git rev-parse --short=7 HEAD) short_sha=$(git rev-parse --short=7 HEAD)
# Dependencies first, so the registry never briefly holds a package
# whose dependency has not landed yet.
dirs=$(node scripts/publish-order.mjs)
for dir in packages/*/; do # Pass 1: stamp every manifest before anything is packed. `pnpm
name=$(node -p "require('./${dir}package.json').name") # publish` substitutes `workspace:*` with the version the linked
base=$(node -p "require('./${dir}package.json').version") # package declares at pack time, so nopy-cube has to be carrying its
# snapshot version by the time nopy is packed.
for dir in $dirs; do
base=$(node -p "require('./${dir}/package.json').version")
# `g` prefix keeps the identifier a valid semver one even when the # `g` prefix keeps the identifier a valid semver one even when the
# abbreviated sha happens to be all digits. # abbreviated sha happens to be all digits.
version="${base}-main.${{ github.run_number }}.g${short_sha}" version="${base}-main.${{ github.run_number }}.g${short_sha}"
(cd "$dir" && npm pkg set "version=${version}")
done
# Pass 2: publish.
for dir in $dirs; do
name=$(node -p "require('./${dir}/package.json').name")
version=$(node -p "require('./${dir}/package.json').version")
echo "::group::${name}@${version}" echo "::group::${name}@${version}"
if npm view "${name}@${version}" version --registry "$REGISTRY" >/dev/null 2>&1; then if npm view "${name}@${version}" version --registry "$REGISTRY" >/dev/null 2>&1; then
echo "Already published — skipping (this is a re-run of the same workflow)." echo "Already published — skipping (this is a re-run of the same workflow)."
else else
( # pnpm, not npm: npm ships `workspace:*` verbatim and the install
cd "$dir" # then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because
npm pkg set "version=${version}" # stamping the versions above left the tree dirty.
npm publish --ignore-scripts --tag main --registry "$REGISTRY" (cd "$dir" && pnpm publish --ignore-scripts --no-git-checks --tag main --registry "$REGISTRY")
)
fi fi
echo "::endgroup::" echo "::endgroup::"
+40 -2
View File
@@ -1,10 +1,15 @@
# Tag-driven release of a single package. # Tag-driven release of a single package.
# #
# git tag nopy-v1.2.0 && git push origin nopy-v1.2.0 # git tag nopy-v1.2.0 && git push origin nopy-v1.2.0
# git tag nopy-cube-v1.2.0 && git push origin nopy-cube-v1.2.0
# git tag keyman-v1.2.0 && git push origin keyman-v1.2.0 # git tag keyman-v1.2.0 && git push origin keyman-v1.2.0
# #
# The tag is the source of truth for *which* package ships; package.json is the # The tag is the source of truth for *which* package ships; package.json is the
# source of truth for the version, and the two must agree or the run fails. # source of truth for the version, and the two must agree or the run fails.
#
# Packages that link to each other release dependency-first — `nopy-cube` before
# `nopy` — because the linked version is resolved at pack time. The run refuses
# to publish otherwise.
# A version with a prerelease part (1.2.0-rc.1) publishes under `next` instead # A version with a prerelease part (1.2.0-rc.1) publishes under `next` instead
# of `latest`. # of `latest`.
# #
@@ -108,6 +113,31 @@ jobs:
- name: Install - name: Install
run: pnpm install --frozen-lockfile run: pnpm install --frozen-lockfile
- name: Check the linked workspace packages are already released
env:
NAME: ${{ steps.target.outputs.name }}
DIR: ${{ steps.target.outputs.dir }}
run: |
set -euo pipefail
# `pnpm publish` turns `workspace:*` into the version the linked
# package declares at this commit. If that version is not on the
# registry yet, the release installs to a broken tree — and npmjs
# only lets you unpublish for 72 hours. Release the dependency first:
# nopy-cube, then nopy, then any bundle.
#
# npmjs only: it is the irreversible one, and it needs no credentials
# to read, which this step does not have yet.
missing=0
for spec in $(node scripts/linked-deps.mjs "$DIR" | tr ' ' '@'); do
if npm view "$spec" version --registry "$NPMJS_REGISTRY" >/dev/null 2>&1; then
echo "${spec} is published"
else
echo "::error::${NAME} depends on ${spec}, which is not on npmjs. Release it first."
missing=1
fi
done
exit "$missing"
- name: Lint - name: Lint
run: pnpm run lint:ci run: pnpm run lint:ci
@@ -121,6 +151,11 @@ jobs:
# Explicit, so the publish steps can skip lifecycle scripts entirely. # Explicit, so the publish steps can skip lifecycle scripts entirely.
run: pnpm run build run: pnpm run build
- name: Verify the packed manifests
# Packages link to each other with `workspace:*`, which npm cannot
# install. Proves on the tarball that pack rewrote it.
run: node scripts/verify-pack.mjs
- name: Publish to the Gitea registry - name: Publish to the Gitea registry
env: env:
NAME: ${{ steps.target.outputs.name }} NAME: ${{ steps.target.outputs.name }}
@@ -139,7 +174,10 @@ jobs:
if npm view "${NAME}@${VERSION}" version --registry "$GITEA_REGISTRY" >/dev/null 2>&1; then if npm view "${NAME}@${VERSION}" version --registry "$GITEA_REGISTRY" >/dev/null 2>&1; then
echo "${NAME}@${VERSION} is already on Gitea — skipping." echo "${NAME}@${VERSION} is already on Gitea — skipping."
else else
(cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --registry "$GITEA_REGISTRY") # pnpm, not npm: npm ships `workspace:*` verbatim and the install
# then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because a
# tag build is a detached HEAD.
(cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --registry "$GITEA_REGISTRY")
fi fi
- name: Publish to npmjs - name: Publish to npmjs
@@ -162,7 +200,7 @@ jobs:
else else
# No --provenance: that needs GitHub Actions OIDC, which Gitea has no # No --provenance: that needs GitHub Actions OIDC, which Gitea has no
# equivalent for. # equivalent for.
(cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY") (cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY")
fi fi
- name: Remove the registry credentials - name: Remove the registry credentials
+2 -1
View File
@@ -1,6 +1,7 @@
{ {
"hosts": [], "hosts": [],
"cubeDirs": ["./cubes"], "cubeDirs": [],
"cubePackages": ["@bitsquare/cubes-core"],
"env": {}, "env": {},
"log": { "log": {
"verbosity": "info", "verbosity": "info",
+144 -32
View File
@@ -4,26 +4,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
## What this repo is ## What this repo is
A pnpm workspace holding two independently published CLIs plus the pyinfra A pnpm workspace holding two independently published CLIs, the authoring package
deployment units one of them runs: their deployment units are written against, and one bundle of those units:
| Path | Package | Binary | Role | | Path | Package | Binary | Role |
| ----------------- | ------------------- | -------- | -------------------------------------------------------- | | --------------------- | ---------------------- | -------- | -------------------------------------------------------- |
| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution | | `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution |
| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` | | `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` |
| `cubes/` | — | — | the deployment units `nopy` runs (not published) | | `packages/nopy-cube` | `@bitsquare/nopy-cube` | — | the authoring surface a `manifest.mjs` imports |
| `packages/cubes-core` | `@bitsquare/cubes-core`| — | the core cube bundle (22 cubes), no TypeScript |
The root package is private; only `packages/*` ship. The two packages do not The root package is private; everything under `packages/` ships. `keyman` stands
depend on each other. alone, but `nopy` and `cubes-core` both depend on `nopy-cube` (`workspace:*`), so
publish order matters — see *Releasing*.
`cubes-core` is consumed the way a third party would consume it: the root
`.nopyrc.json` names it in `cubePackages`, and the loader reads it out of
`node_modules`. There is no `cubes/` directory at the repo root any more.
## Commands ## Commands
```sh ```sh
pnpm install # also installs the git hooks via simple-git-hooks pnpm install # also installs the git hooks via simple-git-hooks
pnpm run build # tsc --build across both packages (project references) pnpm run build # tsc --build across the TS packages (project references)
pnpm run typecheck # tsc --build --noEmit pnpm run typecheck # tsc --build (see below — it really does emit)
pnpm run lint # biome check . (lint:fix / lint:ci variants) pnpm run lint # biome check . (lint:fix / lint:ci variants)
pnpm test # vitest run, both packages pnpm test # vitest run, every package with tests
pnpm run test:coverage # vitest with the coverage gate pnpm run test:coverage # vitest with the coverage gate
pnpm run coverage:summary # renders the last coverage run as a Markdown table pnpm run coverage:summary # renders the last coverage run as a Markdown table
``` ```
@@ -41,6 +47,12 @@ pnpm --filter @bitsquare/keyman run keyman
`typescript` is the 7.x native compiler, so `tsc` *is* the fast one — there is no `typescript` is the 7.x native compiler, so `tsc` *is* the fast one — there is no
separate `tsgo` binary. separate `tsgo` binary.
`typecheck` is plain `tsc --build`, not `--noEmit`. Once a project has
`references`, `--noEmit` is rejected outright (TS6310: *referenced project may
not disable emit*) — a composite project has to emit the declarations its
dependents read. So the typecheck writes `dist` as a side effect; it is
gitignored, and the upside is that the gate now also proves the build works.
## Verification gate ## Verification gate
`lint:ci``typecheck``test:coverage` is one gate, run in three places: the `lint:ci``typecheck``test:coverage` is one gate, run in three places: the
@@ -55,7 +67,14 @@ locally and on the runner. Barrel files (`src/index.ts`, `src/cubes/index.ts`,
`src/nopy.cubes.ts`) and the Commander argv wiring (`src/*.cli.ts`) are excluded; `src/nopy.cubes.ts`) and the Commander argv wiring (`src/*.cli.ts`) are excluded;
adding logic to those files means moving it somewhere covered. adding logic to those files means moving it somewhere covered.
Both packages set `pool: 'forks'` because tests use `process.chdir()` — most nopy's vitest config aliases `@bitsquare/nopy-cube` to that package's **source**,
not to the workspace link (which points at a `dist` that only exists after a
build), so the gate does not depend on build ordering and can never run against a
stale artefact. The same config excludes `**/nopy-cube/**` from coverage — without
it nopy's numbers absorb another package's files. `cubes-core` has no tests of its
own; the loader tests in nopy cover the contract it implements.
The three TS packages set `pool: 'forks'` because tests use `process.chdir()` — most
loader/config tests build a throwaway tree under `os.tmpdir()` and chdir into it, loader/config tests build a throwaway tree under `os.tmpdir()` and chdir into it,
since discovery is driven entirely by the working directory. since discovery is driven entirely by the working directory.
@@ -68,22 +87,43 @@ One pass per invocation, `nopy.main.ts` orchestrating:
Per-property strategy comes from the child's `resolution` block (`merge` is Per-property strategy comes from the child's `resolution` block (`merge` is
the default: arrays concatenate and dedupe, objects deep-merge; `override` the default: arrays concatenate and dedupe, objects deep-merge; `override`
replaces). Only properties listed in `PATH_PROPERTIES` (`cubeDirs`) get replaces). Only properties listed in `PATH_PROPERTIES` (`cubeDirs`) get
relative paths resolved against their own config file's directory. **Throws** relative paths resolved against their own config file's directory;
`cubePackages` needs the same origin for a different reason, so each entry is
normalised into a `CubePackageRef {spec, from}``from` is the directory of
the config that named it, which is where the package gets resolved from.
**Throws**
if no config file exists anywhere — which is why `nopy.cli.ts` calls it lazily if no config file exists anywhere — which is why `nopy.cli.ts` calls it lazily
inside the action, so `--help`/`--version` work outside a project. inside the action, so `--help`/`--version` work outside a project.
2. **`cubes/loader.ts`** — `findCubeDirectories()` unions `config.cubeDirs` with 2. **`cubes/packages.ts`** — `resolveCubePackages()` turns each `CubePackageRef`
every ancestor directory holding a `.npcubes` marker, then scans each into a package root plus the directories its `nopy.cubes` field declares.
recursively (skipping dotted dirs and `node_modules`). A directory is a cube Resolution goes through `createRequire(...).resolve.paths()` + `existsSync`,
deliberately bypassing the `exports` map: a bundle ships directories and has
no entry point to declare. `existsSync` also follows the symlink pnpm plants
at `node_modules/<name>`, which a `readdir` scan skips outright (it reports
`isSymbolicLink()`, not `isDirectory()`). A missing package, an unreadable
manifest, a missing `nopy.cubes`, a directory that does not exist, and an
entry pointing outside the package root are all errors, never silent skips.
Duplicate refs are deduped here, last-wins, because `mergeValue` only dedupes
arrays of primitives and these are objects.
3. **`cubes/loader.ts`** — `findCubeRoots()` unions `config.cubeDirs`, the
directories from `cubePackages`, and every ancestor directory holding a
`.npcubes` marker, then scans each recursively (skipping dotted dirs and
`node_modules`). A directory is a cube
when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a
deploy script (`deploy.py` or `*.deploy.py`); manifests are loaded by dynamic deploy script (`deploy.py` or `*.deploy.py`); manifests are loaded by dynamic
`import()`. Cube id = `manifest.id` → a `[id]` prefix in `manifest.name` `import()`. Cube id = `manifest.id` → a `[id]` prefix in `manifest.name`
the directory basename. Ids are flat and need not mirror the path the directory basename. Ids are flat and need not mirror the path
(`cubes/network/tailscale` declares `net:tailscale`). Duplicate ids and bad (`cubes/network/tailscale` declares `net:tailscale`), and they are claimed
manifests become entries in `errors`, which aborts the run. **globally**, not per source: a duplicate is a hard error naming every
3. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and claimant, with no precedence rule and no shadowing. Each cube carries a
`source``{type: 'dir', dir}` or `{type: 'package', packageName, dir}`
which is what makes that error legible when the collision is between a local
tree and an installed bundle. Duplicate ids and bad manifests become entries
in `errors`, which aborts the run.
4. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and
normalises all three into a `WorkflowResult`. Replays never re-prompt except normalises all three into a `WorkflowResult`. Replays never re-prompt except
for passwords (never persisted) and a missing host. for passwords (never persisted) and a missing host.
4. **`cubes/dependencies.ts``BuildContext.resolveCube()`** — the core. 5. **`cubes/dependencies.ts``BuildContext.resolveCube()`** — the core.
Recursive, per (cube, host): assign params and schema defaults → collect Recursive, per (cube, host): assign params and schema defaults → collect
variables (prompt, or read them back from the session on replay) → run variables (prompt, or read them back from the session on replay) → run
`before` hooks → resolve `manifest.dependencies(vars)` (dynamic: it receives `before` hooks → resolve `manifest.dependencies(vars)` (dynamic: it receives
@@ -92,33 +132,71 @@ One pass per invocation, `nopy.main.ts` orchestrating:
`${cubeId}:${host}` set makes emission idempotent. Hooks get a `HookContext` `${cubeId}:${host}` set makes emission idempotent. Hooks get a `HookContext`
whose `exec(id, vars)` re-enters `resolveCube`, so a hook can pull in a cube whose `exec(id, vars)` re-enters `resolveCube`, so a hook can pull in a cube
that is not a declared dependency. that is not a declared dependency.
5. **`nopy.executor.ts`** — runs the built `pyinfra <host> -y --data K=V ... --chdir <cubeDir> <script>` 6. **`nopy.executor.ts`** — runs the built `pyinfra <host> -y --data K=V ... --chdir <cubeDir> <script>`
commands through execa with inherited stdio, sequentially, stopping at the commands through execa with inherited stdio, sequentially, stopping at the
first failure unless `continueOnError`. first failure unless `continueOnError`.
### Variables ### Variables
`Variables` (`nopy.common.ts`) keeps three per-cube scopes plus one global bag. `Variables` (`nopy.common.ts`) holds one `Variable` per (cube, key). A `Variable`
`get(id)` merges them lowest-to-highest: `global` (from config `env`) → is a list of `Assignment {value, origin}`, and precedence is the `Origin` rank:
`defaults` (Zod `.default()`, and recorded session values on replay) → `prompts` `default(0) < env(1) < session(2) < prompt(3) < param(4)`. There are no scope
(what the user typed) → `params` (values passed by a dependency spec or hook). bags — config `env` is seeded per cube as a real assignment, so the old
`get('global')` (a cube id that was never a cube) is gone, and a replay assigns at
`session` instead of being smuggled into the prompts bag.
`assignments` is the true history, newest first, never reordered. `ordered` is a
**stable** sort of it by rank; `value` / `origin` read its head. The stability is
load-bearing: it is what makes same-origin ties resolve to the newest while the
displaced value stays visible in the trace. The trace is never persisted.
`get(id)` returns the effective values (→ the pyinfra command line);
`persistable(id)` returns the same minus declared secrets (→ session and history).
Every schema key is guaranteed present on the pyinfra side; pyinfra parses Every schema key is guaranteed present on the pyinfra side; pyinfra parses
`--data` values itself, so `"true"` arrives as a bool and numeric strings as ints. `--data` values itself, so `"true"` arrives as a bool and numeric strings as ints.
A manifest's `secrets: string[]` names schema keys holding sensitive values —
validated at load (an entry that is not a schema key aborts the run). Secrets are
excluded from `persistable()`, re-prompted on replay via `fillSessionGaps`
(`requiredKeys() secrets`), and masked by `maskCommand()` / `maskVariables()`
wherever a command is printed. Deliberately a plain array rather than zod
metadata: `.meta()` and `.describe()` live in the per-copy `z.globalRegistry`, so
a manifest built by a different zod copy would look up empty — fail-open is
tolerable for a prompt label and not for a secret marker. See `docs/REFACTORING.md`
items 6 and 7.
### Cube contract ### Cube contract
A cube directory holds `manifest.mjs` + `deploy.py`; anything else in it is A cube directory holds `manifest.mjs` + `deploy.py`; anything else in it is
ignored by the loader but reachable from the script, which runs with the cube ignored by the loader but reachable from the script, which runs with the cube
directory as its cwd. Manifests are ESM, import `cubes.Manifest` from directory as its cwd. Manifests are ESM, import `Manifest` from
`@bitsquare/nopy`, and declare `id`, `name`, a Zod `schema` (each field `@bitsquare/nopy-cube`, and declare `id`, `name`, a Zod `schema` (each field
`.describe()`d — the description is the prompt label — and `.default()`ed), plus `.describe()`d — the description is the prompt label — and `.default()`ed), plus
optional `dependencies`/`before`/`after`. optional `secrets`/`dependencies`/`before`/`after`.
**Gotcha:** those manifests resolve `@bitsquare/nopy` through ordinary Node Import from **`@bitsquare/nopy-cube`**, not `@bitsquare/nopy`. The authoring
resolution from the manifest's own directory. Nothing in this repo links the surface is types and a factory, with zod as its only peer — no CLI, no prompts,
package into `cubes/` or `packages/nopy/cubes/`, so loading them fails with no process spawning — so a bundle can depend on it without dragging the CLI in.
`ERR_MODULE_NOT_FOUND` until you link it (`pnpm --filter @bitsquare/nopy run `@bitsquare/nopy` re-exports all of it (`cubes.Manifest`, `cubes.uniqid`, …), so
link:local`, then `npm link @bitsquare/nopy` where you run from). the older form still works; every cube in `packages/cubes-core` has been moved to
the new one.
Manifests are resolved by ordinary Node resolution **from the manifest's own
directory**, which used to mean a hand-written local cube failed with
`ERR_MODULE_NOT_FOUND` unless you linked the package. `cubes/resolve-hook.mjs`
retires that: `loadCubes()` registers a `module.register()` resolve hook that
tries normal resolution *first* and only on failure falls back to resolving
`@bitsquare/nopy-cube`, `@bitsquare/nopy` and `zod` from the running CLI's own
`node_modules`. Ordinary-resolution-first is the load-bearing part — a cube that
ships its own zod keeps it. The hook is a convenience, never load-bearing:
registration is wrapped in a `try`, and a bundle installed properly never reaches
it. `dist/cubes/*.mjs` is copied by the build, not compiled — hence
`"build": "tsc && cp src/cubes/*.mjs dist/cubes/"`.
Its tests must spawn a real `node` child process. Written inside the vitest
worker they prove nothing: vite resolves the dynamic import itself, so they pass
whether or not the hook is installed — verified by commenting the registration
out and watching them stay green.
## keyman architecture ## keyman architecture
@@ -134,7 +212,7 @@ the environment beats the config file. Encryption shells out to `age` /
Tag-driven, one package at a time; see `README.PUBLISH.md`. Tag-driven, one package at a time; see `README.PUBLISH.md`.
- Push to `main``publish-snapshot.yml` publishes both packages to the Gitea - Push to `main``publish-snapshot.yml` publishes every package to the Gitea
registry as `<version>-main.<run>.g<sha>` under the `main` dist-tag. The registry as `<version>-main.<run>.g<sha>` under the `main` dist-tag. The
version is set on the runner with `npm pkg set` and never committed. version is set on the runner with `npm pkg set` and never committed.
- `git tag <dir>-v<version>` (e.g. `nopy-v1.2.0` — the directory under - `git tag <dir>-v<version>` (e.g. `nopy-v1.2.0` — the directory under
@@ -145,8 +223,42 @@ Tag-driven, one package at a time; see `README.PUBLISH.md`.
So: bump `packages/<pkg>/package.json`, land it on `main`, then tag that commit. So: bump `packages/<pkg>/package.json`, land it on `main`, then tag that commit.
Three things the `workspace:*` links added, all of them non-obvious:
- **`pnpm publish`, never `npm publish`.** `link-workspace-packages` is unset and
pnpm 10+ defaults it to `false`, so `workspace:*` is mandatory in the manifests
— and npm does not understand it. `npm pack` ships the literal string and the
install fails with `EUNSUPPORTEDPROTOCOL`; `pnpm pack`/`pnpm publish` substitute
the real version at pack time. Both directions were measured, not assumed.
- **`scripts/verify-pack.mjs`** packs every non-private package and fails if any
`workspace:` range survived into a tarball. It runs in both publish workflows.
Note `pnpm pack` has no `--ignore-scripts` flag, so `prepack` does rebuild —
which means the artefact under test is the one publish ships.
- **Order.** `packages/*/` sorts `nopy` before `nopy-cube`, which is backwards.
`scripts/publish-order.mjs` topologically sorts over the `workspace:` edges;
the snapshot workflow stamps *every* version first and only then publishes in
that order, because `pnpm publish` reads the linked package's version at pack
time. `release.yml` additionally refuses to ship a package whose linked
dependency is not yet on npmjs (`scripts/linked-deps.mjs`) — npmjs is the
registry you cannot take a mistake back from.
## Known drift ## Known drift
`logConfigToFlags()` is exported and tested but nothing feeds its output into the `logConfigToFlags()` is exported and tested but nothing feeds its output into the
built pyinfra command, so `log.verbosity` / `log.debug` in `.nopyrc.json` built pyinfra command, so `log.verbosity` / `log.debug` in `.nopyrc.json`
currently have no effect. Treat `docs/REFACTORING.md` as a plan, not a record. currently have no effect. Treat `docs/REFACTORING.md` as a plan, not a record.
The publish-lane changes above have been verified locally (pack, npm-install of
the tarballs into a throwaway tree, run) but have **never run against the Gitea
registry**. Burn a throwaway version there before the first real release.
Nothing checks that a bundle and the CLI reading it are compatible versions;
`nopy.engines` was considered and deferred. `docs/CUBE-PACKAGES.md` is where all
of this came from and is now a record of what was built, including what differed
from the plan.
`docs/API.md` predates several refactors and still describes `Cube` and
`Manifest` as plain interfaces with a `key` field; the code has a `Cube` class
keyed on `id`. The `cubePackages` and `CubeSource` sections added for this work
are accurate; treat the rest of that file with suspicion. `DOCS-AUDIT.md` tracks
the wider drift.
+78 -11
View File
@@ -16,7 +16,9 @@ state.
Findings closed since are marked **✅ … fixed** and keep their original text as Findings closed since are marked **✅ … fixed** and keep their original text as
the record of what was wrong. So far: §1.1 (`--use-defaults`), §2.2 the record of what was wrong. So far: §1.1 (`--use-defaults`), §2.2
(`getDefaults()`), half of §2.1 (precedence), and one bullet of §6.4. (`getDefaults()`), §2.1 (precedence — the second half closed differently than
proposed), §4.2 (password on stdout — points 1 and 2 of 3), §4.3 (what a session
records), part of §3.5 (dead exports), and one bullet of §6.4.
--- ---
@@ -138,14 +140,22 @@ dependency detected".
## 2. Documented behaviour that differs from the code ## 2. Documented behaviour that differs from the code
### 2.1 🟡 Variable precedence is wrong in both directions — **partly fixed** ### 2.1 Variable precedence is wrong in both directions — **fixed**
> **Half resolved.** `env` now outranks the Zod defaults, as documented — this > **Resolved, though the second pair closed the opposite way to the README's
> was a prerequisite for `--use-defaults` being configurable at all. The second > original claim.** `env` now outranks the Zod defaults, as documented — that was
> pair was deliberately left: dependency/hook params still outrank prompts. In > a prerequisite for `--use-defaults` being configurable at all.
> practice they do not compete, because `VariableAssignment` leaves out any key >
> a dependency already supplied, so the operator is never asked about it. The > Dependency/hook params still outrank prompts, deliberately. They do not compete
> README now describes that rather than the old claim. > in practice: `VariableAssignment` leaves out any key a dependency supplied, so
> the operator is never asked about it and there is no typed value to override.
> The README documents the real order rather than the old promise.
>
> The underlying complaint — that precedence was the field order of an object
> literal and so could be neither named nor questioned — is what
> `docs/REFACTORING.md` item 6 addresses. `Origin` now ranks
> `default < env < session < prompt < param` as data, and every value carries the
> origin it came from.
`README.md:107-113` stated: `README.md:107-113` stated:
@@ -185,6 +195,12 @@ Two pairs are inverted, and both have consequences:
> schema key rather than only the defaulted ones; and `--use-defaults` refuses > schema key rather than only the defaulted ones; and `--use-defaults` refuses
> to deploy a cube whose required key nothing supplied. Verified against all 22 > to deploy a cube whose required key nothing supplied. Verified against all 22
> cubes in `cubes/`: 19 build a complete `-D` run, the 3 below abort by name. > cubes in `cubes/`: 19 build a complete `-D` run, the 3 below abort by name.
>
> Re-measured against the 25 cubes now in `packages/cubes-core/cubes`: 20 build a
> complete `-D` run and 5 abort by name. Two of the additions are deliberate —
> `user:add` lost its `PUBKEY` default (it was a specific personal key), and
> `ssh:keygen` inherits that failure because it declares `dependencies: () =>
> ['user:add']` and passes no parameters. See §6.6.
`README.md:99` states it outright; `README.md:105` claims defaults ensure "every `README.md:99` states it outright; `README.md:105` claims defaults ensure "every
cube has a predictable starting state". cube has a predictable starting state".
@@ -438,7 +454,11 @@ Public API in `src/index.ts` with no `API.md` entry: the entire history module
`getHistoryPath`, `formatHistoryList`, `HISTORY_FILE`, `DEFAULT_HISTORY_SIZE`, `getHistoryPath`, `formatHistoryList`, `HISTORY_FILE`, `DEFAULT_HISTORY_SIZE`,
plus `HistoryEntry` / `SessionHistory`), `BuildContext`, plus `HistoryEntry` / `SessionHistory`), `BuildContext`,
`runSessionReplayWorkflow`, `getConfigPaths`, `findCubeDirectories`, `getCube`, `runSessionReplayWorkflow`, `getConfigPaths`, `findCubeDirectories`, `getCube`,
`filterInternalVariables`, `separateEnvAndCubeVariables`. ~~`filterInternalVariables`, `separateEnvAndCubeVariables`~~ — those last two were
dead on arrival (nothing called them once the session recorder stopped splitting
`env` out) and have since been deleted rather than documented. `maskCommand`,
`maskVariables`, `Variable`, `Variables`, `MASK` and the `Origin` / `Assignment`
types are newly exported and also have no `API.md` entry.
The CLI cheat-sheet (`API.md:554-578`) omits `-R`, `-H`, `-P`, `--no-history`, The CLI cheat-sheet (`API.md:554-578`) omits `-R`, `-H`, `-P`, `--no-history`,
and the `history` / `clear-history` commands. and the `history` / `clear-history` commands.
@@ -474,7 +494,20 @@ Because `loadCubes` turns each failure into an `errors` entry and `nopy.main.ts:
aborts when `errors.length > 0`, a fresh clone cannot run a single cube. Neither aborts when `errors.length > 0`, a fresh clone cannot run a single cube. Neither
README mentions a setup step. README mentions a setup step.
### 4.2 🟠 The SSH password is printed in plaintext ### 4.2 🟠 The SSH password is printed in plaintext — **mostly fixed**
> **Points 1 and 2 resolved; point 3 stands.** `maskCommand()`
> (`nopy.executor.ts`) rewrites the SSH `--password` and every `--data` value the
> manifest declared a secret, and it is applied at all three places the command
> string is printed: the debug log, the dry-run plan, and `--print-only`. The
> name heuristic described in point 2 is gone — the manifest's `secrets` array
> says which keys are sensitive, so `TOKEN`, `PSK` and `AUTH_KEY` are covered
> too, and it no longer matters that a key merely *looks* like a password.
>
> Point 3 is unchanged and now documented instead: the value still reaches
> pyinfra on its command line, so it is visible in `ps`. That is inherent to
> pyinfra's `--data` interface, not something nopy can mask. The shell-quoting
> concern in the same point is also still open. See `docs/REFACTORING.md` item 7.
Not stated in any document, and it sits directly against the security notes at Not stated in any document, and it sits directly against the security notes at
`README.md:217` and `:325` (which are narrowly about *storage*, and are correct `README.md:217` and `:325` (which are narrowly about *storage*, and are correct
@@ -500,7 +533,18 @@ That string is then:
visible in the process list and, without quoting, vulnerable to shell visible in the process list and, without quoting, vulnerable to shell
metacharacters in the password. metacharacters in the password.
### 4.3 🟠 History and session files record only prompted values ### 4.3 History and session files record only prompted values — **fixed**
> **Resolved.** `buildDeployCall` records `Variables.persistable(cubeId)` — every
> value the cube settled on, whatever its origin — so a `--use-defaults` run no
> longer records an empty object, and a replay reproduces the run rather than
> re-deriving it from whatever the defaults and `env` say at replay time. The one
> deliberate exclusion is a key the manifest declared a secret; those are prompted
> for again on replay, and a `-D` replay that would need one fails by name.
>
> The divergence noted at the end of this finding narrows but does not vanish: a
> dependency graph that resolves differently can still produce a different
> command, because `param` outranks `session` by design.
`README.md:414` says an entry records "the variable values that were answered at `README.md:414` says an entry records "the variable values that were answered at
the prompts" — accurate, but the consequence is not drawn out. the prompts" — accurate, but the consequence is not drawn out.
@@ -671,6 +715,29 @@ the three has to give.
Covered under §1.5. `docs/API.md:160` documents the error; there is no code that Covered under §1.5. `docs/API.md:160` documents the error; there is no code that
raises it. Mutually dependent cubes recurse until the stack overflows. raises it. Mutually dependent cubes recurse until the stack overflows.
### 6.6 🟠 `ssh:keygen` depends on `user:add` but shares nothing with it
`ssh:keygen` declares `dependencies: () => ['user:add']` with no parameters, so
the two cubes resolve their `USER` independently:
- `ssh:keygen` defaults `USER` to `vagrant`;
- `user:add` defaults `USER` to `` user${uniqid(5)} `` — a fresh random name.
So the dependency creates an account the dependent then ignores, and generates a
key for a `vagrant` user it never created. Passing the value through
(`[['user:add', {USER}]]`) is what the dependency spec exists for; `param`
outranks `default`, so it would take effect.
Surfaced by removing `user:add`'s `PUBKEY` default: `ssh:keygen` now fails a `-D`
run with `Cube "user:add" cannot run with --use-defaults: PUBKEY has no default
value`, naming a cube the operator did not select. The underlying mismatch is
older than that change and is not fixed here.
Related: `user:add`'s `USER` default is generated (`` user${uniqid(5)} ``), the
same shape as the `PASSWORD` default that was removed. It is recorded in the
session, so replays are stable, but each fresh `-D` run still creates a
differently-named account.
--- ---
## 7. Checked and accurate ## 7. Checked and accurate
+138 -23
View File
@@ -21,19 +21,44 @@ shipped. If you only want to cut a release, jump to
## What ships ## What ships
| Directory | Package | Binary | | Directory | Package | Binary | Kind |
| ----------------- | ------------------ | -------- | | --------------------- | ----------------------- | -------- | ------------------------ |
| `packages/nopy` | `@bitsquare/nopy` | `nopy` | | `packages/nopy` | `@bitsquare/nopy` | `nopy` | CLI |
| `packages/keyman` | `@bitsquare/keyman` | `keyman` | | `packages/keyman` | `@bitsquare/keyman` | `keyman` | CLI |
| `packages/nopy-cube` | `@bitsquare/nopy-cube` | — | library (cube authoring) |
| `packages/cubes-core` | `@bitsquare/cubes-core` | — | cube bundle (no build) |
Both are ESM, both declare `engines.node >= 22`, and both expose a single All are ESM and declare `engines.node >= 22`. The two CLIs expose a single
executable through `bin`, so `npm install -g` puts `nopy` / `keyman` on the executable through `bin`, so `npm install -g` puts `nopy` / `keyman` on the
`PATH`. `cubes/` is not a package and is never published. `PATH`; the other two are libraries you add to a project.
The tarball contents are pinned by `files: ["dist", "README.md", "LICENSE"]` The tarball contents are pinned by `files` — for the three TypeScript packages
sources and tests are not shipped. `publishConfig.access: "public"` is what makes that is `["dist", "README.md", "LICENSE"]`, so sources and tests are not shipped.
a scoped package publishable to npmjs without an extra flag; the workflows pass `cubes-core` ships `["cubes", "!cubes/**/*.log", "README.md", "LICENSE"]`: the
`--access public` anyway. negation matters, because a cube that has been run leaves a `pyinfra-debug.log`
next to its `deploy.py`, and `.gitignore` does not filter an npm tarball.
`publishConfig.access: "public"` is what makes a scoped package publishable to
npmjs without an extra flag; the workflows pass `--access public` anyway.
### Dependencies between them
`keyman` stands alone. `nopy` and `cubes-core` both depend on `nopy-cube` through
`workspace:*`, which drives three rules the rest of this document keeps coming
back to:
1. **Publish with `pnpm`, not `npm`.** `link-workspace-packages` is unset and
pnpm 10+ defaults it to `false`, so `workspace:*` is mandatory in the
manifests. npm has no idea what that protocol is: `npm pack` copies the string
through verbatim and the install fails with `EUNSUPPORTEDPROTOCOL`. `pnpm
pack` and `pnpm publish` substitute the concrete version at pack time. Both
workflows use `pnpm publish --ignore-scripts --no-git-checks`.
2. **`nopy-cube` publishes before anything that depends on it.**
`node scripts/publish-order.mjs` prints the publishable directories in
dependency order — note that plain alphabetical `packages/*/` gets this
backwards, putting `nopy` first.
3. **Every packed manifest is checked.** `node scripts/verify-pack.mjs` packs
each non-private package and fails if any `workspace:` range survived into the
tarball. It runs in both publish workflows, after the build.
Versions and changelogs are maintained **by hand**. Nothing in CI commits a Versions and changelogs are maintained **by hand**. Nothing in CI commits a
version bump, opens a release PR, or pushes a tag. A release happens because you version bump, opens a release PR, or pushes a tag. A release happens because you
@@ -47,7 +72,7 @@ All three live in [`.gitea/workflows`](.gitea/workflows) and run on the
| Workflow | Trigger | Publishes | | Workflow | Trigger | Publishes |
| ---------------------- | -------------------------------- | ------------------------------------------ | | ---------------------- | -------------------------------- | ------------------------------------------ |
| `ci.yml` | pull requests, non-`main` pushes | nothing | | `ci.yml` | pull requests, non-`main` pushes | nothing |
| `publish-snapshot.yml` | pushes to `main` | **both** packages → Gitea, tag `main` | | `publish-snapshot.yml` | pushes to `main` | **every** package → Gitea, tag `main` |
| `release.yml` | tags matching `*-v*` | **one** package → Gitea **and** npmjs | | `release.yml` | tags matching `*-v*` | **one** package → Gitea **and** npmjs |
`ci.yml` explicitly excludes `main` and all tags (`branches-ignore` + `ci.yml` explicitly excludes `main` and all tags (`branches-ignore` +
@@ -64,13 +89,17 @@ half-finished publish is worse than a slow queue.
``` ```
checkout → pnpm → node → pnpm store cache → install checkout → pnpm → node → pnpm store cache → install
→ lint:ci → typecheck → test:coverage → coverage summary → lint:ci → typecheck → test:coverage → coverage summary
→ build → npm pack --dry-run (per package) → upload coverage → build → npm pack --dry-run (per package) → verify-pack
→ upload coverage
``` ```
The `npm pack --dry-run --ignore-scripts` step prints the exact file list that The `npm pack --dry-run --ignore-scripts` step prints the exact file list that
would be published. It is there to catch a `files` or `bin` entry pointing at would be published. It is there to catch a `files` or `bin` entry pointing at
something the build no longer produces — a failure that would otherwise only something the build no longer produces — a failure that would otherwise only
surface after the version is already on a registry and immutable. surface after the version is already on a registry and immutable. `verify-pack`
then packs for real and checks no `workspace:` range survived; both publish
workflows run it too, but running it here is what puts the failure on the pull
request rather than on the release.
Coverage HTML/JSON reports are uploaded as a `coverage` artifact with a 7-day Coverage HTML/JSON reports are uploaded as a `coverage` artifact with a 7-day
retention. Both the artifact upload and the store cache are retention. Both the artifact upload and the store cache are
@@ -82,18 +111,25 @@ slower CI rather than broken CI.
``` ```
checkout → pnpm → node → cache → install checkout → pnpm → node → cache → install
→ lint:ci → typecheck → test:coverage → coverage summary → build → lint:ci → typecheck → test:coverage → coverage summary → build
→ write .npmrc → publish both packages → delete .npmrc verify-pack → write .npmrc → publish every package → delete .npmrc
``` ```
One job, no `needs:` barrier, so install and build happen exactly once and One job, no `needs:` barrier, so install and build happen exactly once and
nothing has to be passed between jobs as an artifact. nothing has to be passed between jobs as an artifact.
The publish step is **two passes** over `node scripts/publish-order.mjs`: the
first stamps the snapshot version into every manifest with `npm pkg set`, the
second publishes. They cannot be one loop — `pnpm publish` reads a linked
package's version out of its manifest at pack time, so stamping and publishing
one package at a time would bake the *old* `nopy-cube` version into `nopy`'s
tarball.
### `release.yml` ### `release.yml`
``` ```
checkout → resolve tag → check secrets checkout → resolve tag → check secrets
→ pnpm → node → cache → install → pnpm → node → cache → install → check linked deps are released
→ lint:ci → typecheck → test:coverage → build → lint:ci → typecheck → test:coverage → build → verify-pack
→ publish to Gitea → publish to npmjs → delete .npmrc → publish to Gitea → publish to npmjs → delete .npmrc
→ create the Gitea release → step summary → create the Gitea release → step summary
``` ```
@@ -102,6 +138,14 @@ Tag resolution and the secret check run **before** anything is installed or
built, so a malformed tag or a missing token fails in seconds instead of after built, so a malformed tag or a missing token fails in seconds instead of after
the whole gate. the whole gate.
*Check linked deps are released* asks npmjs whether every `workspace:` dependency
of the package being released already exists at the version pnpm is about to
bake in (`scripts/linked-deps.mjs``npm view`). Tagging `nopy-v1.3.0` while
`@bitsquare/nopy-cube@1.1.0` is still unpublished would otherwise ship a tarball
nobody can install, and npmjs only lets you unpublish for 72 hours. The check is
npmjs-only: it runs before any credentials are written, and npmjs is the registry
where the mistake is permanent.
## The verification gate ## The verification gate
The same three commands guard every path into a registry: The same three commands guard every path into a registry:
@@ -143,7 +187,7 @@ a prerelease over `latest` by accident.
## Snapshots ## Snapshots
Every commit that lands on `main` publishes both packages to the Gitea registry, Every commit that lands on `main` publishes every package to the Gitea registry,
versioned as: versioned as:
``` ```
@@ -175,7 +219,34 @@ edit is discarded with the workspace and is never committed.
``` ```
The tag name is `<directory>-v<version>` — the directory under `packages/`, not The tag name is `<directory>-v<version>` — the directory under `packages/`, not
the npm name. `nopy-v1.2.0`, not `@bitsquare/nopy-v1.2.0`. the npm name. `nopy-v1.2.0`, not `@bitsquare/nopy-v1.2.0`. All four prefixes work
the same way:
```sh
git tag nopy-v1.2.0
git tag keyman-v1.2.0
git tag nopy-cube-v1.2.0
git tag cubes-core-v1.2.0
```
### Ordering when more than one package changed
Tags are independent, but the dependency graph is not. If a release touches
`nopy-cube` *and* something that depends on it, release them in this order,
waiting for each run to go green:
```
nopy-cube → nopy, cubes-core (these two are independent of each other)
```
Release `nopy` first and the run stops at the *check linked deps* step, telling
you the `nopy-cube` version it wanted is not on npmjs. That is the guard working;
release `nopy-cube`, then re-tag. `node scripts/publish-order.mjs` prints the
order if you would rather not reason about it.
Bumping `nopy-cube` means bumping the packages that depend on it in the same
change — the `workspace:*` range resolves to whatever version is in the workspace
at pack time, so their next release picks it up whether or not you meant it to.
The tag decides **which** package ships; `package.json` decides the **version**. The tag decides **which** package ships; `package.json` decides the **version**.
The workflow re-reads the manifest and refuses to continue if the two disagree: The workflow re-reads the manifest and refuses to continue if the two disagree:
@@ -281,12 +352,21 @@ file written into the workspace can never be committed by accident.
## Installing the packages ## Installing the packages
From npmjs — public, no configuration: From npmjs — public, no configuration. The CLIs go on the `PATH`:
```sh ```sh
npm install -g @bitsquare/nopy @bitsquare/keyman npm install -g @bitsquare/nopy @bitsquare/keyman
``` ```
The other two go into a project. A cube bundle is a dev dependency of whatever
repo describes your infrastructure; `nopy-cube` is only needed if you are writing
cubes of your own:
```sh
pnpm add -D @bitsquare/cubes-core # then name it in .nopyrc.json cubePackages
pnpm add -D @bitsquare/nopy-cube zod # authoring your own manifests
```
From the Gitea registry, which holds every snapshot plus a mirror of every From the Gitea registry, which holds every snapshot plus a mirror of every
release. Per-project, in the repo's `.npmrc`: release. Per-project, in the repo's `.npmrc`:
@@ -321,9 +401,17 @@ refuses to overwrite an existing version — without the check, a re-run would f
on the first registry and never reach the second. on the first registry and never reach the second.
**The build is explicit, publishes are `--ignore-scripts`.** `prepack` exists for **The build is explicit, publishes are `--ignore-scripts`.** `prepack` exists for
humans running `npm pack` locally; in CI the build has already run as its own humans packing locally; in CI the build has already run as its own step, and
step, and repeating it inside `npm publish` would only cost time and add a way repeating it inside `pnpm publish` would only cost time and add a way for a
for a lifecycle script to change what ships after the gate looked at it. lifecycle script to change what ships after the gate looked at it.
**`verify-pack.mjs` checks the artefact, not the source.** Reading `package.json`
in the repo would only tell you what you already know — every one of them says
`workspace:*`. The question is what pnpm wrote into the tarball, so the script
packs, extracts `package/package.json`, and reads the ranges back out. It cannot
pass `--ignore-scripts`, because `pnpm pack` has no such flag (only `pnpm publish`
does), so `prepack` rebuilds — which at least means the tarball under test is
byte-for-byte the one publish would ship.
**One job per workflow.** No artifact hand-off, no second install, no risk of **One job per workflow.** No artifact hand-off, no second install, no risk of
publishing a tree that a different job built. publishing a tree that a different job built.
@@ -352,7 +440,31 @@ See what would actually be in the tarball:
```sh ```sh
pnpm run build pnpm run build
cd packages/nopy && npm pack --dry-run --ignore-scripts cd packages/nopy && pnpm pack --dry-run
```
Check that no `workspace:` range leaks into a published manifest — the same
check CI runs:
```sh
node scripts/verify-pack.mjs
node scripts/publish-order.mjs # the order to release in
node scripts/linked-deps.mjs packages/nopy # what must be on the registry first
```
Rehearse an install the way a stranger gets one, without publishing anything.
Use **npm**, not pnpm: npm is the one that rejects a leaked `workspace:` range,
so a clean install here is the real proof.
```sh
pnpm --filter @bitsquare/nopy-cube pack --pack-destination /tmp/tgz
pnpm --filter @bitsquare/nopy pack --pack-destination /tmp/tgz
pnpm --filter @bitsquare/cubes-core pack --pack-destination /tmp/tgz
mkdir /tmp/try && cd /tmp/try && npm init -y
npm install /tmp/tgz/*.tgz
echo '{"hosts":["h"],"cubePackages":["@bitsquare/cubes-core"]}' > .nopyrc.json
./node_modules/.bin/nopy install -l session.json -P -D
``` ```
Try the binary as an end user would get it, without publishing: Try the binary as an end user would get it, without publishing:
@@ -384,6 +496,9 @@ npm view @bitsquare/nopy@1.2.0 version \
| `npm pack --dry-run` step fails | A `files` or `bin` path no longer exists after the build. Fix before it reaches a registry. | | `npm pack --dry-run` step fails | A `files` or `bin` path no longer exists after the build. Fix before it reaches a registry. |
| Snapshot workflow green, nothing installable | Snapshots are only on Gitea and only under `@main`. Point the scope at the Gitea registry. | | Snapshot workflow green, nothing installable | Snapshots are only on Gitea and only under `@main`. Point the scope at the Gitea registry. |
| The release workflow did not trigger | The tag must match `*-v*` and must be pushed (`git push origin <tag>`), not just created. | | The release workflow did not trigger | The tag must match `*-v*` and must be pushed (`git push origin <tag>`), not just created. |
| `EUNSUPPORTEDPROTOCOL` / `Unsupported URL Type "workspace:"` on install | A `workspace:` range reached a tarball — something published with `npm publish` instead of `pnpm publish`. `verify-pack.mjs` exists to catch this before it ships. |
| `... is not published yet on npmjs` before the gate runs | Releasing a package before its `nopy-cube` dependency. Tag and release `nopy-cube` first, then re-tag. |
| `verify-pack.mjs` fails locally with a build error | `pnpm pack` runs `prepack`, so a broken build fails the check. Fix the build; there is no skip flag. |
## Recovering from a bad publish ## Recovering from a bad publish
-25
View File
@@ -1,25 +0,0 @@
import { cubes } from '@bitsquare/nopy';
import { z } from 'zod';
export default cubes.Manifest({
id: 'user:add',
name: 'Add a user with fish shell and tools',
dependencies: () => ['apt:essentials'],
schema: z.object({
USER: z
.string()
.describe('Username for the new user account')
.default(() => `user${cubes.uniqid(5)}`),
PASSWORD: z.string().describe('Password for the new user account').default(cubes.uniqid),
GROUPS: z
.string()
.describe('Comma-separated list of additional groups (e.g., "docker,sudo")')
.default(''),
PUBKEY: z
.string()
.describe('SSH public key to authorize for the user')
.default(
'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICpnZ6IxwQKL1rBE4dy7w5Sd3s2tLFZUDfjH87C1QIlc bdiedrichsen@Benjamins-MBP.lan'
),
}),
});
+2 -1
View File
@@ -13,7 +13,7 @@
"test": "pnpm -r run test", "test": "pnpm -r run test",
"test:coverage": "pnpm -r run test:coverage", "test:coverage": "pnpm -r run test:coverage",
"coverage:summary": "node scripts/coverage-summary.mjs", "coverage:summary": "node scripts/coverage-summary.mjs",
"typecheck": "tsc --build --noEmit", "typecheck": "tsc --build",
"lint": "biome check .", "lint": "biome check .",
"lint:fix": "biome check --write .", "lint:fix": "biome check --write .",
"lint:ci": "biome ci .", "lint:ci": "biome ci .",
@@ -26,6 +26,7 @@
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "^2.5.5", "@biomejs/biome": "^2.5.5",
"@bitsquare/cubes-core": "workspace:*",
"@logtape/logtape": "^2.2.4", "@logtape/logtape": "^2.2.4",
"@types/node": "^26.1.1", "@types/node": "^26.1.1",
"simple-git-hooks": "^2.13.1", "simple-git-hooks": "^2.13.1",
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 bitsquare
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+59
View File
@@ -0,0 +1,59 @@
# @bitsquare/cubes-core
The core cube bundle for [nopy](https://www.npmjs.com/package/@bitsquare/nopy):
base packages, users, SSH, firewalling, networking, web serving and runtimes.
## Install
```sh
pnpm add -D @bitsquare/cubes-core
```
Then name it in `.nopyrc.json`:
```json
{
"hosts": ["web-1"],
"cubePackages": ["@bitsquare/cubes-core"]
}
```
`nopy` resolves the package from the directory of the config file that named it,
reads `nopy.cubes` out of its `package.json`, and scans those directories exactly
as it scans a `cubeDirs` entry. Nothing has to be linked or copied.
## What is in it
| Area | Cube ids |
| ---------- | ------------------------------------------------------------------- |
| admin | `admin:cockpit`, `admin:hostname`, `admin:locale` |
| packages | `apt:essentials`, `apt:install` |
| hardening | `armor:fail2ban`, `armor:ssh`, `armor:ufw` |
| web | `caddy`, `caddy:spa` |
| source | `git:clone` |
| networking | `net:tailscale`, `net:wifi:access-point`, `net:wifi:connection` |
| runtimes | `runtime:docker`, `runtime:nodevm` |
| services | `service:autostart` |
| ssh | `ssh:authorize`, `ssh:keygen`, `ssh:keyman` |
| users | `user:add`, `user:edit` |
Run `nopy` and pick from the list, or `nopy -P` to print the pyinfra commands
without executing them. Each cube directory has its own `README.md`.
## Cube ids are global
An id such as `apt:essentials` is claimed repo-wide, not per bundle: two cubes
with the same id — whichever sources they came from — abort the run with an
error naming both. Prefix your own cubes distinctly if you also point
`cubeDirs` at a local tree.
## The bundle is read-only
Under pnpm the installed files are hardlinked into the global store, so a cube
that writes next to its own `deploy.py` corrupts that store for every project on
the machine. Cubes here write to `/tmp` or to the remote host, never to their
own directory.
## License
MIT
@@ -1,6 +1,6 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
export default cubes.Manifest({ export default Manifest({
id: 'admin:cockpit', id: 'admin:cockpit',
name: 'Install cockpit and utils', name: 'Install cockpit and utils',
dependencies: () => [], dependencies: () => [],
@@ -1,11 +1,11 @@
import { cubes, uniqid } from '@bitsquare/nopy'; import { Manifest, uniqid } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
/** /**
* Manifest for the admin:hostname cube. * Manifest for the admin:hostname cube.
* This cube allows for setting and persistently changing the system's hostname. * This cube allows for setting and persistently changing the system's hostname.
*/ */
export default cubes.Manifest({ export default Manifest({
id: 'admin:hostname', id: 'admin:hostname',
name: 'Permanently change the hostname', name: 'Permanently change the hostname',
dependencies: () => [], dependencies: () => [],
@@ -12,9 +12,9 @@ Configures system keyboard layout permanently by updating `/etc/default/keyboard
## Usage ## Usage
```javascript ```javascript
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
export default cubes.Manifest({ export default Manifest({
name: 'My Host Setup', name: 'My Host Setup',
dependencies: () => [ dependencies: () => [
['admin:locale', { LAYOUT: 'de' }] ['admin:locale', { LAYOUT: 'de' }]
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'admin:locale', id: 'admin:locale',
name: 'Configure system locale and keyboard layout', name: 'Configure system locale and keyboard layout',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'apt:essentials', id: 'apt:essentials',
name: 'Install essential packages', name: 'Install essential packages',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'apt:install', id: 'apt:install',
name: 'Install packages with apt', name: 'Install packages with apt',
dependencies: () => [], dependencies: () => [],
@@ -1,6 +1,6 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
export default cubes.Manifest({ export default Manifest({
id: 'armor:fail2ban', id: 'armor:fail2ban',
name: 'Install and enable fail2ban', name: 'Install and enable fail2ban',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'armor:ssh', id: 'armor:ssh',
name: 'Secure SSH server by disabling password authentication', name: 'Secure SSH server by disabling password authentication',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'armor:ufw', id: 'armor:ufw',
name: 'Activate ufw (uncomplicated firewall)', name: 'Activate ufw (uncomplicated firewall)',
dependencies: () => ['apt:essentials'], dependencies: () => ['apt:essentials'],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'caddy', id: 'caddy',
name: 'Install Caddy webserver', name: 'Install Caddy webserver',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'caddy:spa', id: 'caddy:spa',
name: 'Install single page application', name: 'Install single page application',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'git:clone', id: 'git:clone',
name: 'Clone a repository', name: 'Clone a repository',
dependencies: () => [], dependencies: () => [],
@@ -13,7 +13,7 @@ Installs and authenticates the Tailscale client on a Linux host.
| Variable | Default | Description | | Variable | Default | Description |
|----------|---------|-------------| |----------|---------|-------------|
| `AUTH_KEY` | `""` | Tailscale Auth Key (recommended to use a 'reusable' or 'ephemeral' key). | | `AUTH_KEY` | `""` | **Secret.** Tailscale Auth Key (recommended to use a 'reusable' or 'ephemeral' key). |
| `LOGIN_SERVER` | `https://controlplane.tailscale.com` | The coordination server URL. Set this to your Headscale instance URL if applicable. | | `LOGIN_SERVER` | `https://controlplane.tailscale.com` | The coordination server URL. Set this to your Headscale instance URL if applicable. |
| `EXTRA_ARGS` | `""` | Additional flags to pass to `tailscale up` (e.g., `--advertise-exit-node`). | | `EXTRA_ARGS` | `""` | Additional flags to pass to `tailscale up` (e.g., `--advertise-exit-node`). |
| `FORCE_REAUTH` | `false` | If true, forces the client to re-authenticate. | | `FORCE_REAUTH` | `false` | If true, forces the client to re-authenticate. |
@@ -25,3 +25,9 @@ nopy install tailscale
``` ```
When prompted, provide your `AUTH_KEY`. If you are using Headscale, also provide the `LOGIN_SERVER` URL. When prompted, provide your `AUTH_KEY`. If you are using Headscale, also provide the `LOGIN_SERVER` URL.
`AUTH_KEY` is declared in the manifest's `secrets`, so nopy keeps it out of session
and history files and masks it in any command it prints. It is asked for again on
replay, and a `--use-defaults` replay refuses rather than joining the tailnet with
an empty key. Prefer an ephemeral key regardless — the value is still on pyinfra's
command line while the deployment runs.
@@ -1,10 +1,11 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'net:tailscale', id: 'net:tailscale',
name: 'Install and authenticate Tailscale', name: 'Install and authenticate Tailscale',
dependencies: () => ['apt:essentials'], dependencies: () => ['apt:essentials'],
secrets: ['AUTH_KEY'],
schema: z.object({ schema: z.object({
AUTH_KEY: z.string().describe('Tailscale Auth Key for headless authentication').default(''), AUTH_KEY: z.string().describe('Tailscale Auth Key for headless authentication').default(''),
LOGIN_SERVER: z LOGIN_SERVER: z
@@ -19,10 +19,19 @@ Configures a Linux device as a WiFi Access Point using NetworkManager's `nmcli`
## Configuration Parameters ## Configuration Parameters
### Required > **This section is out of date** — it lists parameters the manifest does not
> declare (`NETWORK_DEVICE`, `CHANNEL`, `IP_ADDRESS`) and omits `AP_IP`. Read
> `manifest.mjs` for the real list. Tracked as §5 in the repository's
> `DOCS-AUDIT.md`.
- **SSID**: WiFi network name (1-32 characters) ### Prompted first
- **PASSWORD**: WPA2 password (8-63 characters)
- **SSID**: WiFi network name (1-32 characters). Defaults to `PiPoint`.
- **PASSWORD**: WPA2 password (8-63 characters). Defaults to `1223334444` — a
placeholder that should not survive contact with a real network.
Declared in the manifest's `secrets`, so nopy keeps it out of session and
history files and masks it in printed commands, and re-prompts on replay. The
value is still on pyinfra's command line, so it is visible in `ps` during the run.
### Optional (with defaults) ### Optional (with defaults)
@@ -1,10 +1,11 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'net:wifi:access-point', id: 'net:wifi:access-point',
name: 'Configure WiFi Access Point (NetworkManager)', name: 'Configure WiFi Access Point (NetworkManager)',
dependencies: () => [], dependencies: () => [],
secrets: ['PASSWORD'],
schema: z.object({ schema: z.object({
SSID: z.string().min(1).max(32).default('PiPoint').describe('WiFi network name (SSID)'), SSID: z.string().min(1).max(32).default('PiPoint').describe('WiFi network name (SSID)'),
PASSWORD: z PASSWORD: z
@@ -42,6 +42,11 @@ nopy install network:wifi:connection --env SSID="OfficeWiFi" --env PASSWORD="pas
## Security Notes ## Security Notes
- `PASSWORD` is declared in the manifest's `secrets`: nopy keeps it out of session
and history files and masks it in every command it prints. It is prompted for
again on replay.
- That covers what nopy writes, not everything. The value is still on pyinfra's
command line, so it is visible in `ps` while the deployment runs.
- WiFi passwords will be stored in `/etc/NetworkManager/system-connections/` on the target host. - WiFi passwords will be stored in `/etc/NetworkManager/system-connections/` on the target host.
- Passing passwords via `--env` may leave them in your local shell history. - Passing passwords via `--env` may leave them in your local shell history.
@@ -1,4 +1,4 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
// [agnt://cogen/cogen/network-wifi-connection-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"} // [agnt://cogen/cogen/network-wifi-connection-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"}
@@ -7,9 +7,10 @@ import { z } from 'zod';
* Manifest for the network:wifi:connection cube. * Manifest for the network:wifi:connection cube.
* Configures a WiFi client connection using NetworkManager (nmcli). * Configures a WiFi client connection using NetworkManager (nmcli).
*/ */
export default cubes.Manifest({ export default Manifest({
id: 'net:wifi:connection', id: 'net:wifi:connection',
name: 'network:wifi:connection - Connect to a WiFi network', name: 'network:wifi:connection - Connect to a WiFi network',
secrets: ['PASSWORD'],
schema: z.object({ schema: z.object({
SSID: z.string().min(1).describe('The SSID of the WiFi network to connect to'), SSID: z.string().min(1).describe('The SSID of the WiFi network to connect to'),
PASSWORD: z.string().min(8).describe('The password for the WiFi network'), PASSWORD: z.string().min(8).describe('The password for the WiFi network'),
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'runtime:docker', id: 'runtime:docker',
name: 'Install docker and tools', name: 'Install docker and tools',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'runtime:nodevm', id: 'runtime:nodevm',
name: 'Install nvm and nodejs with global packages', name: 'Install nvm and nodejs with global packages',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'service:autostart', id: 'service:autostart',
name: 'Manage systemd service autostart', name: 'Manage systemd service autostart',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'ssh:authorize', id: 'ssh:authorize',
name: 'Authorize SSH public key for a user', name: 'Authorize SSH public key for a user',
dependencies: () => [], dependencies: () => [],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'ssh:keygen', id: 'ssh:keygen',
name: 'Generate SSH key for a given $USER', name: 'Generate SSH key for a given $USER',
dependencies: () => ['user:add'], dependencies: () => ['user:add'],
@@ -1,7 +1,7 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
export default cubes.Manifest({ export default Manifest({
id: 'ssh:keyman', id: 'ssh:keyman',
name: 'Deploy an ssh key managed by keyman', name: 'Deploy an ssh key managed by keyman',
dependencies: () => [], dependencies: () => [],
@@ -36,9 +36,15 @@ This cube creates a new user account with a modern shell environment (Fish), SSH
- Username for the new user account - Username for the new user account
- Default: `userXXXXX` (randomly generated 5-character suffix) - Default: `userXXXXX` (randomly generated 5-character suffix)
- **PASSWORD** (string, auto-generated) - **PASSWORD** (string, **secret**)
- Password for the new user account - Password for the new user account
- Default: randomly generated secure password - Default: the literal `changeme` — a placeholder, not a credential. Change it
on first login, or pass a real one.
- Declared in the manifest's `secrets`, so it is never written to a session or
history file and is masked in printed commands. A replay asks for it again.
- It used to default to a randomly generated password. That was removed: since
the value is not recorded, an unattended run created an account with a
credential nobody had seen, and replaying that run produced a different one.
- **GROUPS** (string, default: `''`) - **GROUPS** (string, default: `''`)
- Comma-separated list of additional groups (e.g., `"docker,sudo"`) - Comma-separated list of additional groups (e.g., `"docker,sudo"`)
@@ -47,9 +53,16 @@ This cube creates a new user account with a modern shell environment (Fish), SSH
- `sudo` - Administrative privileges - `sudo` - Administrative privileges
- `www-data` - Web server file access - `www-data` - Web server file access
- **PUBKEY** (string, has default) - **PUBKEY** (string, **required** — no default)
- SSH public key to authorize for the user - SSH public key to authorize for the user
- Should be your public key for passwordless SSH access - Should be your public key for passwordless SSH access
- There is deliberately no default. It used to be a specific personal key, so
accepting the default authorized *someone else's* key on the new account.
No key would be a sensible guess, so the cube asks instead.
- Because it is required, `--use-defaults` refuses to run this cube unless
`PUBKEY` comes from `env` in `.nopyrc.json`, a dependency, or a hook.
- Submitting an empty value at the prompt authorizes no key at all (the account
is still created, with password login only).
## Dependencies ## Dependencies
@@ -7,7 +7,10 @@ USER = host.data.USER
HOME_DIR = f"/home/{USER}" HOME_DIR = f"/home/{USER}"
TMP_DIR = f"{HOME_DIR}/tmp" TMP_DIR = f"{HOME_DIR}/tmp"
PASSWORD = host.data.PASSWORD PASSWORD = host.data.PASSWORD
# An empty submission at the prompt must not become an empty authorized_keys
# line, so an absent key means no key rather than a blank one.
PUBKEY = host.data.PUBKEY PUBKEY = host.data.PUBKEY
PUBKEYS = [PUBKEY] if PUBKEY and str(PUBKEY).strip() else []
GROUPS = list(filter(None, map(str.strip, str(host.data.GROUPS).split()))) GROUPS = list(filter(None, map(str.strip, str(host.data.GROUPS).split())))
FISH_PATH = "/usr/bin/fish" FISH_PATH = "/usr/bin/fish"
FISH_CONFIG_DIR = f"{HOME_DIR}/.config/fish" FISH_CONFIG_DIR = f"{HOME_DIR}/.config/fish"
@@ -31,7 +34,7 @@ server.user(
create_home=True, create_home=True,
groups=GROUPS, groups=GROUPS,
shell=FISH_PATH, shell=FISH_PATH,
public_keys=[PUBKEY], public_keys=PUBKEYS,
_sudo=True _sudo=True
) )
@@ -0,0 +1,28 @@
import { Manifest, uniqid } from '@bitsquare/nopy-cube';
import { z } from 'zod';
export default Manifest({
id: 'user:add',
name: 'Add a user with fish shell and tools',
dependencies: () => ['apt:essentials'],
secrets: ['PASSWORD'],
schema: z.object({
USER: z
.string()
.describe('Username for the new user account')
.default(() => `user${uniqid(5)}`),
// A fixed placeholder, not a generated one: the password is never recorded
// in a session, so a generated default meant every run produced credentials
// nobody had seen and a replay produced different ones again.
PASSWORD: z.string().describe('Password for the new user account').default('changeme'),
GROUPS: z
.string()
.describe('Comma-separated list of additional groups (e.g., "docker,sudo")')
.default(''),
// No default on purpose. This used to carry a specific personal key, which
// meant an unattended run authorised someone else's key on the new account.
// Leaving it required makes `--use-defaults` refuse by name instead of
// guessing, and there is no key that would be a sensible guess.
PUBKEY: z.string().describe('SSH public key to authorize for the user'),
}),
});
@@ -18,7 +18,7 @@ This cube allows you to update existing user accounts on the target system. It c
| Variable | Type | Description | Required | | Variable | Type | Description | Required |
| :--- | :--- | :--- | :--- | | :--- | :--- | :--- | :--- |
| `USER` | `string` | The username of the account to modify | Yes | | `USER` | `string` | The username of the account to modify | Yes |
| `PASSWORD` | `string` | New password for the user | No | | `PASSWORD` | `string` | New password for the user. **Secret**: never recorded in a session or history file, masked in printed commands, re-prompted on replay. | No |
| `GROUPS` | `string` | Comma-separated list of groups to ADD (e.g., `docker,sudo`) | No | | `GROUPS` | `string` | Comma-separated list of groups to ADD (e.g., `docker,sudo`) | No |
| `GROUPS_ABSENT` | `string` | Comma-separated list of groups to REMOVE | No | | `GROUPS_ABSENT` | `string` | Comma-separated list of groups to REMOVE | No |
@@ -1,4 +1,4 @@
import { cubes } from '@bitsquare/nopy'; import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod'; import { z } from 'zod';
// [agnt://cogen/cogen/user-edit-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"} // [agnt://cogen/cogen/user-edit-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"}
@@ -7,10 +7,11 @@ import { z } from 'zod';
* Manifest for the user:edit cube. * Manifest for the user:edit cube.
* Allows modifying existing user accounts (password, groups). * Allows modifying existing user accounts (password, groups).
*/ */
export default cubes.Manifest({ export default Manifest({
id: 'user:edit', id: 'user:edit',
name: 'user:edit - Modify an existing user account', name: 'user:edit - Modify an existing user account',
dependencies: () => [], dependencies: () => [],
secrets: ['PASSWORD'],
schema: z.object({ schema: z.object({
USER: z.string().describe('The username of the account to modify'), USER: z.string().describe('The username of the account to modify'),
PASSWORD: z.string().optional().describe('New password for the user (optional)'), PASSWORD: z.string().optional().describe('New password for the user (optional)'),
+45
View File
@@ -0,0 +1,45 @@
{
"name": "@bitsquare/cubes-core",
"version": "1.0.0-alpha0",
"description": "The core nopy cube bundle: apt, users, ssh, networking, services and runtimes.",
"keywords": [
"nopy",
"nopy-cubes",
"pyinfra",
"deployment",
"infrastructure"
],
"license": "MIT",
"author": "bitsquare",
"type": "module",
"repository": {
"type": "git",
"url": "git+https://gitea.bitsquare.dev/BitSquare/ansiblings.git",
"directory": "packages/cubes-core"
},
"homepage": "https://gitea.bitsquare.dev/BitSquare/ansiblings/src/branch/main/packages/cubes-core",
"bugs": {
"url": "https://gitea.bitsquare.dev/BitSquare/ansiblings/issues"
},
"engines": {
"node": ">=22"
},
"nopy": {
"cubes": [
"./cubes"
]
},
"files": [
"cubes",
"!cubes/**/*.log",
"README.md",
"LICENSE"
],
"publishConfig": {
"access": "public"
},
"dependencies": {
"@bitsquare/nopy-cube": "workspace:*",
"zod": "^4.4.3"
}
}
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 bitsquare
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+69
View File
@@ -0,0 +1,69 @@
# @bitsquare/nopy-cube
The authoring surface for [nopy](https://www.npmjs.com/package/@bitsquare/nopy)
cubes — the `Manifest` factory, the `Cube` class, and the types around them.
A cube manifest ships nothing but data, so it should not have to depend on a CLI
to describe itself. This package is what a **cube bundle** depends on: no
`commander`, no `inquirer`, no `execa`, no process spawning. `@bitsquare/nopy`
re-exports everything here, so a manifest that already imports from
`@bitsquare/nopy` keeps working unchanged.
## Install
```sh
pnpm add @bitsquare/nopy-cube zod
```
`zod` is a **peer dependency** on purpose: the manifest, the schema it builds and
the `Manifest` factory should all see the same copy.
## Writing a manifest
```js
// cubes/net/tailscale/manifest.mjs
import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod';
export default Manifest({
id: 'net:tailscale',
name: 'Tailscale',
schema: z.object({
AUTH_KEY: z.string().describe('Tailscale auth key'),
ACCEPT_ROUTES: z.boolean().describe('Accept advertised routes').default(true),
}),
secrets: ['AUTH_KEY'],
dependencies: (vars) => (vars.ACCEPT_ROUTES ? ['net:ip-forwarding'] : []),
before: [async (ctx, vars) => ctx.exec('apt:essentials', {})],
});
```
Every schema field should carry a `.describe()` — nopy uses it as the prompt
label — and a `.default()` wherever a sensible one exists, so `--use-defaults`
can run the cube without prompting.
`secrets` names the schema keys that hold sensitive values. Nopy keeps those out
of session and history files and masks them in every command it prints; it does
not infer them, so a key nothing declares is recorded and printed in the clear.
Each entry must be a key of `schema` — naming anything else is a manifest error.
Give a secret a placeholder `.default()` rather than a real credential: a default
lives in the manifest, where none of that protection reaches it.
The manifest lives next to a `deploy.py` in the same directory; together they
make a cube. See the
[nopy README](https://www.npmjs.com/package/@bitsquare/nopy) for the full cube
contract and for how to publish a directory of cubes as a bundle.
## Exports
| Export | What it is |
| ----------------------------------- | -------------------------------------------------------------- |
| `Manifest(opts)` | Builds a manifest, filling in `id`, `schema`, `secrets`, `before`, `after` |
| `createManifest` / `manifest` | Aliases of `Manifest` |
| `Cube` | A loaded manifest plus its directory; `getDefaults()`, `requiredKeys()`, `secrets`, `isSecret()` |
| `zodKind` / `zodInner` | Instance-agnostic zod introspection, safe across zod copies |
| `AnyObjectSchema`, `CubeVariables`, `DependencySpec`, `Hook`, `HookContext`, `CubeSource`, `LoadResult` | types |
## License
MIT
+60
View File
@@ -0,0 +1,60 @@
{
"name": "@bitsquare/nopy-cube",
"version": "1.0.0-alpha0",
"description": "Authoring types for nopy cubes: the Manifest factory and the Cube contract.",
"keywords": [
"nopy",
"pyinfra",
"deployment",
"infrastructure"
],
"license": "MIT",
"author": "bitsquare",
"type": "module",
"repository": {
"type": "git",
"url": "git+https://gitea.bitsquare.dev/BitSquare/ansiblings.git",
"directory": "packages/nopy-cube"
},
"homepage": "https://gitea.bitsquare.dev/BitSquare/ansiblings/src/branch/main/packages/nopy-cube",
"bugs": {
"url": "https://gitea.bitsquare.dev/BitSquare/ansiblings/issues"
},
"engines": {
"node": ">=22"
},
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"./package.json": "./package.json"
},
"files": [
"dist",
"README.md",
"LICENSE"
],
"publishConfig": {
"access": "public"
},
"scripts": {
"clean": "rm -rf dist .tsbuildinfo",
"build": "tsc",
"prepack": "pnpm run build",
"link:local": "pnpm run build && npm link",
"test": "vitest run",
"test:coverage": "vitest run --coverage",
"test:watch": "vitest"
},
"peerDependencies": {
"zod": "^4.4.3"
},
"devDependencies": {
"@types/node": "^26.1.1",
"@vitest/coverage-v8": "^4.1.10",
"typescript": "^7.0.2",
"vitest": "^4.1.10",
"zod": "^4.4.3"
}
}
@@ -1,6 +1,6 @@
/** /**
* Factory functions for creating cube configurations * Factory functions for creating cube configurations
* @module cubes/factories * @module factories
*/ */
import { type AnyObjectSchema, Manifest } from './types.js'; import { type AnyObjectSchema, Manifest } from './types.js';
+31
View File
@@ -0,0 +1,31 @@
/**
* @bitsquare/nopy-cube — the authoring surface for nopy cubes.
*
* Everything a `manifest.mjs` needs and nothing else: no CLI, no prompts, no
* process spawning. `@bitsquare/nopy` re-exports all of it, so a manifest can
* import from either package.
*
* @packageDocumentation
*/
export {
createManifest,
ManifestFactory,
manifest,
} from './factories.js';
export type {
AnyObjectSchema,
CubeSource,
CubeVariables,
DependencySpec,
Hook,
HookContext,
LoadResult,
} from './types.js';
export {
Cube,
Manifest,
zodInner,
zodKind,
} from './types.js';
export { uniqid } from './utils.js';
@@ -1,6 +1,6 @@
/** /**
* Type definitions for Nopy cubes * Type definitions for Nopy cubes
* @module cubes/types * @module types
*/ */
import { z } from 'zod'; import { z } from 'zod';
@@ -47,6 +47,18 @@ export interface Manifest<Schema extends AnyObjectSchema = AnyObjectSchema> {
name: string; name: string;
/** Zod schema for validating cube variables */ /** Zod schema for validating cube variables */
schema: Schema; schema: Schema;
/**
* Schema keys holding secrets. Their values are never written to a session
* file, and are masked wherever a command or a variable would be printed.
*
* A plain array rather than schema-level metadata on purpose: `.meta()` and
* `.describe()` both store into zod's global registry, which is per-copy a
* manifest that builds its schema with its own zod writes the marker into a
* registry this process cannot read. A missed `.describe()` costs an ugly
* prompt label; a missed secret marker writes a password to disk, so this one
* cannot be allowed to fail open. See {@link zodKind} for the same hazard.
*/
secrets?: string[];
/** Dynamic dependency resolver based on collected variables */ /** Dynamic dependency resolver based on collected variables */
dependencies?: (variables: z.infer<Schema>) => DependencySpec[]; dependencies?: (variables: z.infer<Schema>) => DependencySpec[];
/** Hooks to run before cube execution */ /** Hooks to run before cube execution */
@@ -65,6 +77,7 @@ export function Manifest<Schema extends AnyObjectSchema>(
id: opts.id ?? '', id: opts.id ?? '',
name: opts.name, name: opts.name,
schema: opts.schema ?? (z.object({}) as unknown as Schema), schema: opts.schema ?? (z.object({}) as unknown as Schema),
secrets: opts.secrets ?? [],
dependencies: opts.dependencies, dependencies: opts.dependencies,
before: opts.before ?? [], before: opts.before ?? [],
after: opts.after ?? [], after: opts.after ?? [],
@@ -189,6 +202,15 @@ export class Cube<Schema extends AnyObjectSchema = AnyObjectSchema> {
.filter(([, zodType]) => !zodType.safeParse(undefined).success) .filter(([, zodType]) => !zodType.safeParse(undefined).success)
.map(([key]) => key); .map(([key]) => key);
} }
/** Schema keys the manifest declared as secrets. */
get secrets(): string[] {
return this.manifest.secrets ?? [];
}
isSecret(key: string): boolean {
return this.secrets.includes(key);
}
} }
/** /**
@@ -1,6 +1,6 @@
/** /**
* Utility functions for cubes * Utility functions for cubes
* @module cubes/utils * @module utils
*/ */
/** /**
@@ -1,10 +1,10 @@
/** /**
* Tests for cubes/factories module * Tests for the manifest factories
*/ */
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { z } from 'zod'; import { z } from 'zod';
import { createManifest, manifest } from '../src/cubes/factories.js'; import { createManifest, manifest } from '../src/factories.js';
describe('createManifest', () => { describe('createManifest', () => {
it('creates manifest with basic properties', () => { it('creates manifest with basic properties', () => {
@@ -0,0 +1,29 @@
/**
* A schema that behaves like zod's but does not share zod's prototypes.
*
* Once cubes arrive from `node_modules`, the schema a manifest builds may come
* from a *second* copy of zod — its own dependency, or one shipped inside a
* bundle. Such a schema is structurally identical and `instanceof` blind to it.
* Rebuilding the nodes as plain objects reproduces that from inside a single
* process, so anything that reads zod's internals stays pinned to `def.type`.
*/
import type { z } from 'zod';
/** Strips the prototype off a schema node and everything it wraps. */
function strip(node: unknown): unknown {
const def = { ...(node as { def: Record<string, unknown> }).def };
if (def.innerType) def.innerType = strip(def.innerType);
return { def };
}
export function foreignZodSchema<S extends z.ZodObject<any>>(schema: S): S {
return {
// Parsing is not what is under test — delegate it and keep the real
// behaviour, so only the introspection path sees the foreign nodes.
safeParse: (value: unknown) => schema.safeParse(value),
shape: Object.fromEntries(
Object.entries(schema.shape).map(([key, node]) => [key, strip(node)])
),
} as unknown as S;
}
@@ -5,12 +5,39 @@
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { z } from 'zod'; import { z } from 'zod';
import { Cube, Manifest } from '../src/cubes/types.js'; import { Cube, Manifest } from '../src/types.js';
import { foreignZodSchema } from './helpers/foreign-zod.js'; import { foreignZodSchema } from './helpers/foreign-zod.js';
const cube = (schema: z.ZodObject<any>) => const cube = (schema: z.ZodObject<any>) =>
new Cube(Manifest.create({ id: 'c', name: 'C', schema }), '/cubes/c', 'deploy.py'); new Cube(Manifest.create({ id: 'c', name: 'C', schema }), '/cubes/c', 'deploy.py');
describe('Cube', () => {
it('reads id and name off the manifest', () => {
const c = cube(z.object({}));
expect(c.id).toBe('c');
expect(c.name).toBe('C');
});
it('defaults its source to its own directory', () => {
// What the loader overrides when a cube arrives from a package; a cube
// built by hand still has to answer the question.
expect(cube(z.object({})).source).toEqual({ type: 'dir', dir: '/cubes/c' });
});
it('keeps the source it was constructed with', () => {
const source = { type: 'package' as const, packageName: '@acme/cubes-net', dir: '/pkg/cubes' };
const c = new Cube(
Manifest.create({ id: 'c', name: 'C' }),
'/pkg/cubes/c',
'deploy.py',
source
);
expect(c.source).toBe(source);
});
});
describe('Cube.getDefaults', () => { describe('Cube.getDefaults', () => {
it('resolves every default when the whole schema parses', () => { it('resolves every default when the whole schema parses', () => {
const c = cube( const c = cube(
@@ -109,3 +136,36 @@ describe('Cube.requiredKeys', () => {
expect(c.requiredKeys()).toEqual([]); expect(c.requiredKeys()).toEqual([]);
}); });
}); });
describe('Cube.secrets', () => {
it('is empty when the manifest declares none', () => {
const c = cube(z.object({ PASSWORD: z.string().default('x') }));
expect(c.secrets).toEqual([]);
// No name-based guessing: only what the manifest says.
expect(c.isSecret('PASSWORD')).toBe(false);
});
it('reports what the manifest declared', () => {
const c = new Cube(
Manifest.create({
id: 'c',
name: 'C',
schema: z.object({ USER: z.string(), PASSWORD: z.string() }),
secrets: ['PASSWORD'],
}),
'/cubes/c',
'deploy.py'
);
expect(c.secrets).toEqual(['PASSWORD']);
expect(c.isSecret('PASSWORD')).toBe(true);
expect(c.isSecret('USER')).toBe(false);
});
it('defaults to an empty list on a manifest built by hand', () => {
const c = new Cube({ id: 'c', name: 'C', schema: z.object({}) }, '/cubes/c', 'deploy.py');
expect(c.secrets).toEqual([]);
});
});
@@ -1,9 +1,9 @@
/** /**
* Tests for cubes/utils module * Tests for the uniqid helper
*/ */
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { uniqid } from '../src/cubes/utils.js'; import { uniqid } from '../src/utils.js';
describe('uniqid', () => { describe('uniqid', () => {
it('generates string of default length (5)', () => { it('generates string of default length (5)', () => {
+15
View File
@@ -0,0 +1,15 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"tsBuildInfoFile": ".tsbuildinfo",
"outDir": "dist",
"rootDir": "src",
"lib": ["ES2020"],
"composite": true,
"module": "NodeNext",
"types": ["node"]
},
"include": ["src/**/*"],
"exclude": ["coverage", "node_modules", "dist"],
"references": []
}
+25
View File
@@ -0,0 +1,25 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
environment: 'node',
include: ['src/**/*.test.ts', 'tests/**/*.test.ts'],
coverage: {
provider: 'v8',
reporter: ['text', 'json-summary', 'html'],
include: ['src/**/*.ts'],
exclude: [
'src/**/*.test.ts',
// Pure re-export barrel: no logic to cover.
'src/index.ts',
],
thresholds: {
branches: 85,
functions: 85,
lines: 80,
statements: 80,
},
},
},
});
+112 -17
View File
@@ -33,7 +33,7 @@ Nopy wraps pyinfra with structure, validation, and an interactive experience for
A cube is a **directory** containing two files: A cube is a **directory** containing two files:
- **JavaScript manifest**: `manifest.mjs` defining schema, dependencies, defaults, and hooks - **JavaScript manifest**: `manifest.mjs` defining schema, dependencies, defaults, secrets, and hooks
- **Python deployment script**: `deploy.py`, a plain pyinfra script - **Python deployment script**: `deploy.py`, a plain pyinfra script
Configuration variables are declared in the manifest and validated with Zod schemas before the deployment script runs. Configuration variables are declared in the manifest and validated with Zod schemas before the deployment script runs.
@@ -96,7 +96,7 @@ apt.packages(
) )
``` ```
Every key defined in the manifest `schema` is guaranteed to be present on `host.data` — either from the Zod `.default()`, from `.nopyrc.json`, from a dependency, or from a user prompt. Every key defined in the manifest `schema` is guaranteed to be present on `host.data` — either from the Zod `.default()`, from `.nopyrc.json`, from a recorded session, from a dependency, or from a user prompt.
**Value types**: pyinfra parses `--data` values before your script sees them. `"true"` / `"false"` become booleans, numeric strings become `int`, valid JSON becomes the parsed structure, and everything else stays a string. This is why `UPDATE` can be handed straight to pyinfra's `update=` argument, while `PACKAGES` is wrapped in `str(...)` before splitting. **Value types**: pyinfra parses `--data` values before your script sees them. `"true"` / `"false"` become booleans, numeric strings become `int`, valid JSON becomes the parsed structure, and everything else stays a string. This is why `UPDATE` can be handed straight to pyinfra's `update=` argument, while `PACKAGES` is wrapped in `str(...)` before splitting.
@@ -104,18 +104,57 @@ Every key defined in the manifest `schema` is guaranteed to be present on `host.
Variable defaults are defined directly in the Zod schema using `.default()`. This ensures that every cube has a predictable starting state and provides type-safe default values. Variable defaults are defined directly in the Zod schema using `.default()`. This ensures that every cube has a predictable starting state and provides type-safe default values.
**Priority order (lowest to highest):** A variable can be set from several places in one run. Every assignment is kept, tagged with where it came from — its **origin** — and the highest-ranked origin wins.
1. Zod schema `.default()` values **Origins, lowest to highest:**
2. Global `env` from `.nopyrc.json`
3. User prompts, or the recorded answers on session replay | Origin | Set by |
4. Variables passed in by a dependency or a hook | --------- | ------------------------------------------------------- |
| `default` | the Zod schema's `.default()` |
| `env` | the `env` block of `.nopyrc.json` |
| `session` | a value recorded in a session file or history entry |
| `prompt` | what the user typed |
| `param` | a dependency spec or a `before`/`after` hook |
This allows cubes to ship with reasonable defaults while still allowing users to override them globally via `.nopyrc.json` or interactively during deployment. Because `env` outranks the schema, `.nopyrc.json` is also what steers a run started with `--use-defaults`, which never prompts. This allows cubes to ship with reasonable defaults while still allowing users to override them globally via `.nopyrc.json` or interactively during deployment. Because `env` outranks the schema, `.nopyrc.json` is also what steers a run started with `--use-defaults`, which never prompts.
3 and 4 rarely compete: a key a dependency supplies is left out of the prompt entirely, so the user is only ever asked about the keys nothing else has set. `prompt` and `param` rarely compete: a key a dependency supplies is left out of the prompt entirely, so the user is only ever asked about the keys nothing else has set.
A field declared without `.default()` has none of sources 1 and 2 to fall back on. It is prompted for like any other, with an empty initial value — but a run that cannot prompt (`--use-defaults`) fails on it unless `env` or a dependency provides it. Ranking by origin rather than by arrival order is what makes replay work: a recorded value is applied *before* the cube would be prompted for, and prompting can still override it, but a `--data` value pushed in by a dependency is never clobbered by a stale recording.
A field declared without `.default()` has no `default` origin to fall back on. It is prompted for like any other, with an empty initial value — but a run that cannot prompt (`--use-defaults`) fails on it unless `env` or a dependency provides it.
#### Secrets
A manifest can name schema keys that hold sensitive values:
```javascript
export default cubes.Manifest({
id: 'user:add',
name: 'Add a user account',
secrets: ['PASSWORD'],
schema: z.object({
USERNAME: z.string().describe('Username for the new account').default('deploy'),
PASSWORD: z.string().describe('Password for the new user account').default('changeme'),
})
})
```
Every entry must be a key of `schema`; naming anything else is a manifest error and aborts the run, so a typo fails loudly instead of silently leaving a value unprotected.
Declaring a key a secret changes three things:
- **It is never written to a session file or to the history.** Everything else the run settled on is recorded — including values that came from a `.default()` — but declared secrets are left out.
- **It is masked wherever a command or a plan is printed** — `--dry-run`, `--print-only`, and the debug log all show `********` in place of the value, in the variable list *and* in the `pyinfra` command line above it. The SSH password passed via `--password` is masked the same way, whether or not any cube declares secrets.
- **It is re-prompted on replay**, since there is nothing recorded to replay from (see [Session Recording and Replay](#session-recording-and-replay)).
Nopy does not guess. A key called `PASSWORD` in a manifest that declares no `secrets` is treated as an ordinary variable — recorded, and printed in the clear.
Three limits are worth knowing, because `secrets` keeps a value out of the files nopy writes and nothing more:
- **It is on the command line.** pyinfra takes its data as `--data KEY=value`, so the real value is visible in `ps` for as long as the deployment runs. Masking covers nopy's own output, not the process table.
- **The prompt shows it.** The variable form displays and pre-fills what it is asking about, so a secret is on screen while it is being entered or confirmed.
- **A `.default()` is not protected.** A default lives in the manifest, in plain text, wherever the manifest is checked in. Give a secret a placeholder default like `changeme` if it needs one at all, never a real credential.
### Configuration ### Configuration
@@ -125,6 +164,7 @@ Uses `.nopyrc.json` files (project-level or home directory) containing:
{ {
"hosts": ["host1.example.com", "host2.example.com"], "hosts": ["host1.example.com", "host2.example.com"],
"cubeDirs": ["./cubes", "../shared-cubes"], "cubeDirs": ["./cubes", "../shared-cubes"],
"cubePackages": ["@bitsquare/cubes-core"],
"env": { "env": {
"SHARED_VAR": "value" "SHARED_VAR": "value"
}, },
@@ -144,6 +184,8 @@ Uses `.nopyrc.json` files (project-level or home directory) containing:
`history` controls automatic session recording (see [Deployment History](#deployment-history)), and `execution.continueOnError` sets the default for `--continue-on-error`. `history` controls automatic session recording (see [Deployment History](#deployment-history)), and `execution.continueOnError` sets the default for `--continue-on-error`.
`cubeDirs` holds paths, `cubePackages` holds installed npm packages that ship cubes — see [Cube Discovery](#cube-discovery) below and [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md) for publishing your own. Both are additive, and both resolve relative to the config file that named them, not to the working directory: a `.nopyrc.json` two levels up may name a package that only exists in *its* `node_modules`.
#### Logging Configuration #### Logging Configuration
Control pyinfra output verbosity and debug information using the `log` configuration object: Control pyinfra output verbosity and debug information using the `log` configuration object:
@@ -213,12 +255,16 @@ Sessions are stored in `.nopysession.json` files with the following structure:
**Structure Details:** **Structure Details:**
- **`cubes`**: Array of cubes with only cube-specific variables (not global env vars) - **`cubes`**: Array of cubes with the variable values that cube ran with
- **`env`**: Global environment variables shared across cubes (like in `.nopyrc.json`) - **`env`**: The `env` block of `.nopyrc.json` as it stood at record time, kept for reference
- **`hosts`**: Array of target hosts - **`hosts`**: Array of target hosts
- **`auth`**: Authentication configuration (passwords are never stored) - **`auth`**: Authentication configuration (passwords are never stored)
**Security Note**: Passwords are never stored in session files. If a session uses password authentication, you'll be prompted for the password during replay. **What is recorded:** every value each cube settled on, regardless of where it came from — a value the user typed, one inherited from `.nopyrc.json` `env`, one a dependency supplied, and one that fell through to the schema's `.default()` are all written out the same way. A session is therefore a full snapshot rather than a diff, and a `--use-defaults` run produces a session with real values in it instead of an empty one.
The consequence is that replay is faithful rather than re-derived: the recorded value outranks the current `.nopyrc.json` `env` and the current schema default, so editing either one does not silently change what a replay does. To pick up a new default, record a fresh session.
**Security Note**: Passwords are never stored in session files. This covers both the SSH password — a session records the auth *method* and username, never the credential — and any schema key a cube's manifest lists under [`secrets`](#secrets). Both are re-prompted on replay.
#### Recording a Session #### Recording a Session
@@ -240,12 +286,48 @@ nopy install --load-session my-deployment.nopysession.json
# Only password authentication will prompt for credentials # Only password authentication will prompt for credentials
``` ```
A replay runs straight through without asking anything, with three exceptions. Password authentication always re-prompts. A session with no recorded host falls back to the host picker. And a cube is re-prompted for its declared secrets, plus for any required variable the session has no value for — which happens when the cube's schema has gained a field since the session was written.
Those re-prompts are what a session cannot supply, so `--use-defaults` cannot paper over them: combining `-D` with a replay that needs either fails with a message naming the keys rather than deploying with a placeholder. Put the values under `env` in `.nopyrc.json` to make such a replay unattended.
### Cube Discovery ### Cube Discovery
Nopy searches for cubes in: Nopy searches for cubes in:
1. Directories specified in `.nopyrc.json` `cubeDirs` 1. Directories specified in `.nopyrc.json` `cubeDirs`
2. Directories containing a `.npcubes` marker file (searching upwards from current directory) 2. Cube directories of every package listed in `.nopyrc.json` `cubePackages`
3. Directories containing a `.npcubes` marker file (searching upwards from current directory)
All three are unioned and scanned the same way. A directory is a cube when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a deploy script (`deploy.py` or `*.deploy.py`); dotted directories and `node_modules` are skipped during the scan.
#### Cube packages
A cube package is an ordinary npm package that ships cube directories and points at them from its own `package.json`:
```json
{
"name": "@bitsquare/cubes-core",
"nopy": { "cubes": ["./cubes"] }
}
```
Install it and name it — nothing needs linking or copying:
```sh
pnpm add -D @bitsquare/cubes-core
```
```json
{ "cubePackages": ["@bitsquare/cubes-core"] }
```
Naming a package is a statement that cubes are expected from it, so anything wrong is an error that aborts the run rather than a silent skip: the package is not installed, it declares no `nopy.cubes`, or an entry points at a directory that does not exist or lies outside the package.
#### Ids are claimed globally
A cube id such as `apt:essentials` is claimed across every source at once, not per directory or per package. Two cubes with the same id abort the run with an error naming both and where each came from. There is no precedence rule and no shadowing — a local cube does not quietly win over a packaged one, in either direction. Prefix your own cubes distinctly if you point `cubeDirs` at a local tree alongside an installed bundle.
Writing cubes to publish is covered in [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md).
## Command Line Usage ## Command Line Usage
@@ -319,6 +401,17 @@ have no default values. Set them under "env" in .nopyrc.json, pass them from a
dependency, or drop --use-defaults to be prompted. dependency, or drop --use-defaults to be prompted.
``` ```
Pairing `-D` with a replay fails the same way when the replay would have to ask
something — a declared [secret](#secrets), which is never recorded, or a required
variable the session has no value for. Both are the sources `-D` has no substitute
for, so it stops rather than deploying a placeholder:
```
Error: Cube "user:add" cannot be replayed with --use-defaults: PASSWORD would
have to be entered. Secrets are never recorded in a session. Replay without
--use-defaults, or set the values under "env" in .nopyrc.json.
```
**Use SSH key authentication**: **Use SSH key authentication**:
```bash ```bash
@@ -434,20 +527,20 @@ Session History:
Total: 2 session(s) Total: 2 session(s)
``` ```
Each entry records the selected cubes together with the variable values that were answered at the prompts, the target hosts, the authentication method, and the username — never the password. Pass an ID to `-H` to run that exact combination again: Each entry records the selected cubes together with every variable value they ran with, the target hosts, the authentication method, and the username — never the password, and never a key the manifest declared a [secret](#secrets). Pass an ID to `-H` to run that exact combination again:
```bash ```bash
nopy install -H mdk0zzp8b71cq nopy install -H mdk0zzp8b71cq
``` ```
A replay is non-interactive: cube selection, host, and variable values all come from the entry, so nopy runs straight through without asking anything. The two exceptions are password authentication, which always re-prompts, and an entry with no recorded host, which falls back to the host picker. A replay is non-interactive: cube selection, host, and variable values all come from the entry, so nopy runs straight through without asking anything. It asks only for what the entry cannot hold — the password under password authentication, and any declared secret — plus the host picker when the entry recorded none.
Two things are worth knowing before relying on an older entry: Two things are worth knowing before relying on an older entry:
- **Recorded values are applied as defaults, not as a frozen snapshot.** If a cube's schema has gained a variable since the run was recorded, the replay neither prompts for it nor fails — the new variable quietly takes its Zod `.default()`. Global `env` values are likewise read from the *current* `.nopyrc.json` rather than from the entry. - **Recorded values win over the current configuration.** The entry is a snapshot of everything the run settled on, so editing a cube's `.default()` or the `env` block of `.nopyrc.json` afterwards does not change what the replay does. A variable the schema has gained *since* the entry was written has nothing recorded: if it has a `.default()` the replay quietly takes it, and if it is required the replay prompts for it.
- **A replay fails if a cube no longer exists.** Renaming or deleting a cube id makes every history entry that referenced it unreplayable: nopy logs `Cube from session not found` and then aborts with `Cube not found: <id>`. - **A replay fails if a cube no longer exists.** Renaming or deleting a cube id makes every history entry that referenced it unreplayable: nopy logs `Cube from session not found` and then aborts with `Cube not found: <id>`.
The history lives in `.nopy.history.json` in the working directory and uses the same structure as a session file, so trimming the array by hand is a perfectly good way to prune it. It does contain the variable values that were entered, which is why it is listed in this repository's `.gitignore` — treat it like any other file holding deployment configuration. A corrupt or unreadable history file is treated as empty rather than raising an error, which looks exactly like a project that has never been deployed from. The history lives in `.nopy.history.json` in the working directory and uses the same structure as a session file, so trimming the array by hand is a perfectly good way to prune it. It does contain the variable values a run used, which is why it is listed in this repository's `.gitignore` — treat it like any other file holding deployment configuration. A corrupt or unreadable history file is treated as empty rather than raising an error, which looks exactly like a project that has never been deployed from.
For a run you want to keep indefinitely, don't rely on history — it rotates. Use `--save-session` to write it to a file you control (see [Session Recording and Replay](#session-recording-and-replay)). For a run you want to keep indefinitely, don't rely on history — it rotates. Use `--save-session` to write it to a file you control (see [Session Recording and Replay](#session-recording-and-replay)).
@@ -468,7 +561,9 @@ npm run debug
## Documentation ## Documentation
- [Cube Hooks](docs/HOOKS.md) - Lifecycle hooks for dynamic orchestration - [Cube Hooks](docs/HOOKS.md) - Lifecycle hooks for dynamic orchestration
- [Cube Bundles](docs/CUBE-BUNDLES.md) - Distributing cubes as npm packages
- [Session Format](docs/SESSION_FORMAT.md) - Internal JSON/MJS session structure - [Session Format](docs/SESSION_FORMAT.md) - Internal JSON/MJS session structure
- [API Reference](docs/API.md) - Types and exported functions
## Resources ## Resources
+71 -1
View File
@@ -65,6 +65,20 @@ interface NopyResult {
The cubes module provides types and functions for working with deployment units. The cubes module provides types and functions for working with deployment units.
The authoring half of it — `Manifest`, `Cube`, `Hook`, `uniqid` and the rest —
actually lives in **[`@bitsquare/nopy-cube`](../../nopy-cube)**, a package with
no CLI and no dependency other than zod. `@bitsquare/nopy` re-exports all of it,
so both of these work:
```javascript
import { Manifest } from '@bitsquare/nopy-cube'; // in a manifest.mjs — prefer this
import { cubes } from '@bitsquare/nopy'; // cubes.Manifest — still supported
```
Import from `nopy-cube` in a cube bundle you intend to publish: it lets the
bundle depend on the authoring types without pulling the whole CLI in as a
dependency. See [CUBE-BUNDLES.md](CUBE-BUNDLES.md).
### Types ### Types
#### `Cube<Schema>` #### `Cube<Schema>`
@@ -76,6 +90,7 @@ interface Cube<Schema extends z.AnyZodObject = z.AnyZodObject> {
key: string; // Unique identifier key: string; // Unique identifier
name: string; // Human-readable name name: string; // Human-readable name
dir: string; // Absolute path to cube directory dir: string; // Absolute path to cube directory
source: CubeSource; // Where it was discovered
dependencies: string[]; dependencies: string[];
schema: Schema; schema: Schema;
defaults: () => z.infer<Schema>; defaults: () => z.infer<Schema>;
@@ -84,6 +99,18 @@ interface Cube<Schema extends z.AnyZodObject = z.AnyZodObject> {
} }
``` ```
#### `CubeSource`
Where a cube came from. Carried so that a duplicate-id error can name the origin
of each claimant, which is the difference between a usable error message and a
puzzle when the collision is between a local tree and an installed bundle.
```typescript
type CubeSource =
| { type: 'dir'; dir: string }
| { type: 'package'; packageName: string; dir: string };
```
#### `Manifest<Schema>` #### `Manifest<Schema>`
Cube manifest (used in `manifest.mjs` files). Cube manifest (used in `manifest.mjs` files).
@@ -124,7 +151,9 @@ interface HookContext {
#### `loadCubes()` #### `loadCubes()`
Loads all cubes from discovered cube directories. Loads all cubes from discovered cube directories`cubeDirs`, the directories
declared by every package in `cubePackages`, and any ancestor directory holding a
`.npcubes` marker.
```typescript ```typescript
const { cubes, errors } = await loadCubes(); const { cubes, errors } = await loadCubes();
@@ -139,6 +168,27 @@ interface LoadResult {
} }
``` ```
`errors` is non-empty for a duplicate id, a manifest that fails to load, a
package in `cubePackages` that is not installed or declares no cubes, and a
`nopy.cubes` entry that is missing or points outside its package. Any of them
aborts the run — none is a silent skip.
#### `resolveCubePackages(refs)`
Resolves `CubePackageRef[]` to installed packages and their cube directories.
Called by `loadCubes()`; exported because the resolution failures are worth
testing on their own.
```typescript
const { packages, errors } = resolveCubePackages(config.cubePackages);
interface CubePackage {
name: string; // the name it was requested under
root: string; // absolute path to the package root
dirs: string[]; // absolute paths from its `nopy.cubes` field
}
```
#### `resolveDependencies(cubes, selectedCubeNames)` #### `resolveDependencies(cubes, selectedCubeNames)`
Resolves all transitive dependencies for selected cubes. Resolves all transitive dependencies for selected cubes.
@@ -452,11 +502,31 @@ Configuration file structure.
interface NopyConfig { interface NopyConfig {
hosts: string[]; hosts: string[];
cubeDirs: string[]; cubeDirs: string[];
cubePackages: CubePackageRef[];
env: EnvConfig; env: EnvConfig;
log?: LogConfig; log?: LogConfig;
} }
``` ```
#### `CubePackageRef`
A package named in `cubePackages`, paired with where it was named. In the config
file an entry is just a string (`"@bitsquare/cubes-core"`); `loadConfig()`
normalises it.
```typescript
interface CubePackageRef {
/** The package name, as written in the config. */
spec: string;
/** Directory of the config file that named it — resolution starts here. */
from: string;
}
```
`from` is what makes a package named in a parent config resolve against *that*
config's `node_modules`, not the working directory's. It is the same problem
`PATH_PROPERTIES` solves for relative `cubeDirs`.
#### `LogConfig` #### `LogConfig`
Logging configuration. Logging configuration.
+263
View File
@@ -0,0 +1,263 @@
# Cube bundles
How to package cubes as an npm package so other projects can install them, and
what changes once a cube lives in `node_modules` instead of in your own tree.
If you only want to *use* a published bundle, you need one line of config:
```json
{ "cubePackages": ["@bitsquare/cubes-core"] }
```
The rest of this document is for writing one.
- [What a bundle is](#what-a-bundle-is)
- [The package manifest](#the-package-manifest)
- [Writing the cubes](#writing-the-cubes)
- [Ids are claimed globally](#ids-are-claimed-globally)
- [An installed bundle is read-only](#an-installed-bundle-is-read-only)
- [How resolution actually works](#how-resolution-actually-works)
- [Publishing](#publishing)
- [Troubleshooting](#troubleshooting)
## What a bundle is
An ordinary npm package that ships cube directories and points at them from its
own `package.json`. There is no build step, no plugin API and no entry point —
nopy reads the directories off disk and imports each `manifest.mjs` directly.
```
@acme/cubes-web
├── package.json nopy.cubes → ["./cubes"]
├── README.md
└── cubes/
├── nginx/
│ ├── manifest.mjs
│ └── deploy.py
└── certbot/
├── manifest.mjs
└── deploy.py
```
`@bitsquare/cubes-core` in this repository is the worked example, and is consumed
by this repository through exactly the mechanism described here — it is not
special-cased.
## The package manifest
```json
{
"name": "@acme/cubes-web",
"version": "1.0.0",
"type": "module",
"nopy": { "cubes": ["./cubes"] },
"files": ["cubes", "!cubes/**/*.log", "README.md", "LICENSE"],
"publishConfig": { "access": "public" },
"dependencies": {
"@bitsquare/nopy-cube": "^1.0.0",
"zod": "^4.4.3"
}
}
```
**`nopy.cubes`** is the only field nopy requires. It is an array of directories,
relative to the package root, each scanned recursively for cubes. Several
entries are fine; a single `["./cubes"]` is the norm. Every entry must exist and
must stay inside the package — a path escaping the root is refused, not resolved.
**`type: "module"`** matters: manifests are ESM. Without it a `manifest.mjs` still
loads (the extension carries the day), but anything it imports relatively will
not behave the way you expect.
**`files`** decides the tarball. Note the negation: a cube that has been run
leaves a `pyinfra-debug.log` next to its `deploy.py`, and `.gitignore` has no
effect on what npm packs. Check with `npm pack --dry-run` before publishing.
**Dependencies** are `@bitsquare/nopy-cube` and `zod`, both real dependencies
rather than peers — a bundle is a leaf, and the copies it gets are the copies its
manifests use. Do **not** depend on `@bitsquare/nopy`: the CLI is what installs
your bundle, not the other way round, and depending on it invites two copies of
the same code into one tree.
## Writing the cubes
A cube directory holds a manifest (`manifest.mjs` or `*.manifest.mjs`) and a
deploy script (`deploy.py` or `*.deploy.py`). Anything else in the directory is
invisible to the loader but readable from the script, which runs with the cube
directory as its working directory.
```javascript
// cubes/nginx/manifest.mjs
import { Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod';
export default Manifest({
id: 'web:nginx',
name: 'Install and configure nginx',
dependencies: () => ['apt:essentials'],
secrets: ['TLS_KEY'],
schema: z.object({
SERVER_NAME: z.string().describe('Server name').default('example.com'),
TLS_KEY: z.string().describe('TLS private key (PEM)').default(''),
HTTP2: z.boolean().describe('Enable HTTP/2').default(true),
}),
});
```
```python
# cubes/nginx/deploy.py
from pyinfra import host
from pyinfra.operations import apt, files
SERVER_NAME = host.data.SERVER_NAME
apt.packages(name='Install nginx', packages=['nginx'], _sudo=True)
```
Import **`@bitsquare/nopy-cube`**, not `@bitsquare/nopy`. It is types and a
factory with zod as its only peer — no CLI, no prompts, no process spawning — so
your bundle stays a leaf. (`@bitsquare/nopy` re-exports the same surface as
`cubes.Manifest`, which is what older manifests use. It still works; it just
drags the CLI into your dependency graph if you declare it.)
Four things the schema is load-bearing for:
- **`.describe()` is the prompt label.** A field without one prompts with its raw
key.
- **`.default()` makes the field optional.** A field with no default is required,
and is re-prompted on replay if a session has no value for it.
- **Every schema key reaches pyinfra** as `--data KEY=value`, so `host.data.KEY`
is always defined. pyinfra parses the values itself: `"true"` arrives as a
bool, `"8080"` as an int.
- **`secrets` names keys whose values must not be persisted.** They are excluded
from session files and history, masked wherever a command is printed, and
re-prompted on replay. Naming a key that is not in the schema is a load error.
A secret is still visible in `ps` while pyinfra runs — masking covers nopy's
own output, not the process table — so treat it as protection against writing
credentials to disk, not as protection against a shared host.
`dependencies` is a function of the *collected* variables, so it can branch on
what the user actually answered, and it may pass parameters:
```javascript
dependencies: (v) => (v.HTTP2 ? ['apt:essentials', ['web:tls', { MODE: 'strict' }]] : []),
```
`before` / `after` hooks get a context whose `exec(id, vars)` pulls in any cube
by id, declared dependency or not. See [HOOKS.md](HOOKS.md).
## Ids are claimed globally
An id is claimed across every source at once — `cubeDirs`, `.npcubes` trees and
every installed bundle share one flat namespace. Two cubes claiming the same id
abort the run with an error naming both and where each came from.
There is no precedence and no shadowing, deliberately, in either direction: a
local cube does not quietly win over a packaged one, and installing a second
bundle cannot silently change what an existing id deploys. Overriding a cube from
a bundle is not a supported operation; fork the cube under your own id instead.
So prefix distinctly. `@acme/cubes-web` claiming `nginx` is asking for trouble the
first time someone installs a second bundle; `web:nginx` is not. Ids need not
mirror the directory layout — `cubes/network/tailscale` declares `net:tailscale`
— so the prefix is free.
An id is also the session key. Renaming one silently invalidates every recorded
session that used it, so treat a rename as a breaking change of the bundle.
## An installed bundle is read-only
Under pnpm, installed files are **hardlinked into a global store shared by every
project on the machine**. A cube that writes next to its own `deploy.py` does not
just dirty one `node_modules` — it corrupts that store for every other project.
Write to `/tmp`, to a path the user configured, or to the remote host. Never to
the cube's own directory. Files the cube needs to *read* (templates, config
fragments, systemd units) are fine and are exactly what the cube directory is for
`deploy.py` runs with it as the working directory, so `files.template('nginx.conf.j2', ...)`
resolves.
This is the one constraint that does not exist while the cubes live in your own
repo, which makes it the one most likely to be discovered late. Test against an
installed copy, not a linked one.
## How resolution actually works
Worth knowing, because two of the failure modes are otherwise baffling.
**Where a package is looked up from.** Each `cubePackages` entry is resolved from
the directory of the config file that named it, not from the working directory.
Configs merge upward, so a `.nopyrc.json` two levels up can name a bundle that
only exists in *its* `node_modules`, and it resolves. The lookup reads
`package.json` off disk via `createRequire(...).resolve.paths()` rather than
going through `exports` — a bundle ships directories and has no entry point to
declare.
**Why the loader does not simply scan `node_modules`.** It cannot: pnpm plants a
symlink at `node_modules/<name>`, and `readdir` reports it as a symlink, not a
directory, so a recursive scan skips every package silently. Naming packages
explicitly is the fix, and it is also the reason `node_modules` is skipped during
the cube scan itself.
**How a manifest finds its imports.** Ordinary Node resolution, from the
manifest's own directory. An installed bundle has its own `node_modules` with
`@bitsquare/nopy-cube` and `zod` in it, so this just works. A hand-written cube
sitting in a directory with no `node_modules` would historically fail with
`ERR_MODULE_NOT_FOUND`; nopy now registers a resolve hook that catches exactly
that case and falls back to resolving `@bitsquare/nopy-cube`, `@bitsquare/nopy`
and `zod` from the running CLI. Normal resolution is always tried first, so a
cube that ships its own zod keeps it. Treat the hook as a convenience for local
cubes — a published bundle must declare its dependencies properly.
**Two copies of zod is a real hazard.** `instanceof` comparisons fail across
copies, which is why nopy inspects schemas structurally (`schema.def.type`) and
why `secrets` is a plain array rather than `.meta()` metadata — zod's metadata
registry is per-copy, and a marker written into one copy's registry is invisible
to another. Keep your zod range compatible with the CLI's (`^4.4.3`) and the
package manager will usually give you one copy.
## Publishing
Nothing bundle-specific: `npm publish` (or `pnpm publish`) with a version bump.
Some things worth deciding once:
- **Version the bundle independently of nopy.** There is no compatibility check
between the two — the loader reads whatever `nopy.cubes` points at. Document
the nopy version you test against in your README.
- **Renaming or removing an id is breaking.** It invalidates recorded sessions
and breaks any manifest listing it as a dependency, including manifests in
other people's bundles.
- **Changing a schema key is breaking** in the same way; adding one with a
`.default()` is not.
- **Test the installed shape, not the linked one.** `npm pack`, install the
tarball into a throwaway directory with a `.nopyrc.json` naming it, and deploy
from it. This is what catches a missing file, a cube that writes to its own
directory, and an undeclared dependency — none of which show up while the
package is symlinked into the repo that wrote it.
For an unattended check, replay a session file rather than reaching for `-P`
alone, which still opens the interactive picker:
```sh
nopy install -l session.json -P -D
```
Note that a replay re-prompts for anything a manifest lists in `secrets` —
those are never written to a session — so pick a cube without them, or put the
values under `env` in `.nopyrc.json`.
For how this repository releases its own packages, see
[README.PUBLISH.md](../../../README.PUBLISH.md).
## Troubleshooting
| Symptom | Cause |
| --- | --- |
| `Cube package 'X' is not installed (looked up from …)` | Not installed, or installed somewhere other than the config that named it. The path in the message is where the lookup started. |
| `Cube package 'X' declares no cubes` | Missing or malformed `nopy.cubes` in the package's `package.json`. It must be a non-empty array of strings. |
| `'./cubes' does not exist in …` | The directory was not packed. Check `files` and `npm pack --dry-run`. |
| `'…' points outside the package` | A `nopy.cubes` entry escaping the package root. Not allowed. |
| `Duplicate cube id 'X' from N sources:` | Two or more cubes claiming one id; the message lists each source. Rename one — there is no precedence rule to lean on. |
| `ERR_MODULE_NOT_FOUND` for `zod` or `@bitsquare/nopy-cube` | The bundle did not declare them as dependencies. The resolve-hook fallback covers loose local cubes, not published packages. |
| `Invalid manifest in …: 'secrets' names X, which is not in the schema` | A `secrets` entry with no matching schema key — usually a typo or a renamed field. |
| Cubes work linked, fail installed | Almost always a write into the cube's own directory, or a file missing from `files`. |
+154 -45
View File
@@ -1,6 +1,11 @@
# Cube bundles as npm packages # Cube bundles as npm packages
Status: **Phase 0 has landed; Phases 16 are still a plan, not a record.** Status: **All six phases have landed. This document is now a record, not a plan.**
The one thing still unproven is the publish lane against a real registry — see
*Risks*.
`cubePackages` resolves and loads end to end, `@bitsquare/nopy-cube` exists and
the publish lane can ship a linked package. What is missing is a bundle to point
`cubePackages` at.
Distributing cubes as npm packages so a project can `pnpm add @acme/cubes-net` Distributing cubes as npm packages so a project can `pnpm add @acme/cubes-net`
and have its cubes show up in `nopy` alongside local ones. and have its cubes show up in `nopy` alongside local ones.
@@ -166,7 +171,7 @@ Rules:
- A bundle must not ship a `.nopyrc.json`. Config discovery walks up from - A bundle must not ship a `.nopyrc.json`. Config discovery walks up from
`process.cwd()`, never from cube directories, so it would never be read. `process.cwd()`, never from cube directories, so it would never be read.
## Phase 2 — resolution ## Phase 2 — resolution — **done**
### Config surface ### Config surface
@@ -271,7 +276,7 @@ it is exported from `src/cubes/index.ts` and covered by tests. The
`node_modules` skip inside `scanDirectory` stays and is now *correct*: a `node_modules` skip inside `scanDirectory` stays and is now *correct*: a
bundle's own `node_modules` should not be scanned. bundle's own `node_modules` should not be scanned.
## Phase 3 — hard errors with attribution ## Phase 3 — hard errors with attribution — **done**
`Cube` gains a source, as an optional fourth constructor parameter so the public `Cube` gains a source, as an optional fourth constructor parameter so the public
signature stays backwards compatible: signature stays backwards compatible:
@@ -306,7 +311,7 @@ claim the same id they are mutually exclusive, and the fix is upstream.
Surface the source in the interactive picker and in `--json` output so a user can Surface the source in the interactive picker and in `--json` output so a user can
see where a cube came from before running it. see where a cube came from before running it.
## Phase 4 — `@bitsquare/nopy-cube`, the authoring package ## Phase 4 — `@bitsquare/nopy-cube`, the authoring package — **done**
The problem: a manifest does `import { cubes } from '@bitsquare/nopy'`, resolved The problem: a manifest does `import { cubes } from '@bitsquare/nopy'`, resolved
by ordinary Node resolution from the manifest's own directory. From inside by ordinary Node resolution from the manifest's own directory. From inside
@@ -350,20 +355,33 @@ already coverage-excluded barrels — so `import { cubes } from '@bitsquare/nopy
in every existing manifest keeps working unchanged. Nothing in `cubes/` has to be in every existing manifest keeps working unchanged. Nothing in `cubes/` has to be
touched at migration time. touched at migration time.
Repo plumbing this requires: `cubes/types.ts` and `cubes/factories.ts` moved wholesale, with
`tests/cubes.types.test.ts` and `tests/cubes.factories.test.ts` behind them.
`tests/helpers/foreign-zod.ts` is duplicated rather than shared — fifteen lines,
and the alternative is a test-only dependency edge between the packages.
- `tsconfig.base.json`: add `"@bitsquare/nopy-cube": ["./packages/nopy-cube/src"]` Repo plumbing it took:
to `paths`.
- Root `tsconfig.json`: add the project reference. - `tsconfig.base.json`: `"@bitsquare/nopy-cube": ["./packages/nopy-cube/src"]`.
- `packages/nopy/tsconfig.json`: `references` is currently `[]` — add - Root `tsconfig.json` and `packages/nopy/tsconfig.json`: the project reference.
`{ "path": "../nopy-cube" }`. This is the first reference edge in the repo, so This is the first reference edge in the repo, and it broke the gate
`tsc --build` ordering starts mattering. immediately: **`tsc --build --noEmit` is not legal once a project has
references** — TS6310, "referenced project may not disable emit", because a
composite project has to emit the declarations its dependents read. The root
`typecheck` script is now plain `tsc --build`. It still fails on a type error,
and it now also proves the build works; the cost is that it writes `dist`,
which is gitignored.
- `packages/nopy/package.json`: `"@bitsquare/nopy-cube": "workspace:*"`. - `packages/nopy/package.json`: `"@bitsquare/nopy-cube": "workspace:*"`.
- A `vitest.config.ts` for the new package with the same thresholds. `Manifest()`, - `packages/nopy/vitest.config.ts`: a `resolve.alias` for `@bitsquare/nopy-cube`
`Manifest.create()` and `Cube.getDefaults()` all carry logic, so the relevant pointing at `../nopy-cube/src/index.ts`. Without it the workspace link
cases move over from `tests/cubes.factories.test.ts`. resolves through `exports` to `dist`, so `pnpm test` on a clean checkout would
fail until something had built it, and a stale `dist` would silently be what
the tests ran against. The same config excludes `**/nopy-cube/**` from
coverage — the aliased files were being counted against nopy's thresholds.
- A `vitest.config.ts` for the new package with the same thresholds. It sits at
100 % statements/functions/lines, 91 % branches.
### The release lane needs fixing first ### The release lane needed fixing first
This is the part that is easy to miss. `link-workspace-packages` is unset and This is the part that is easy to miss. `link-workspace-packages` is unset and
pnpm 10+ defaults it to `false`, so a plain semver range would resolve pnpm 10+ defaults it to `false`, so a plain semver range would resolve
@@ -389,10 +407,40 @@ Pick one before publishing anything:
passes — compute every snapshot version first, then publish — so `nopy` can pin passes — compute every snapshot version first, then publish — so `nopy` can pin
the exact `nopy-cube` snapshot from the same run. the exact `nopy-cube` snapshot from the same run.
Recommendation: `pnpm publish`, and verify against the Gitea registry with a **Measured, both directions.** `npm pack` in `packages/nopy` produces a tarball
whose manifest still reads `"@bitsquare/nopy-cube": "workspace:*"`; `pnpm pack`
produces one that reads `"1.0.0-alpha0"`. So the failure was real and the fix
works.
Went with `pnpm publish --ignore-scripts --no-git-checks` in both workflows.
`--no-git-checks` is not optional in either: `release.yml` runs on a detached
HEAD, and `publish-snapshot.yml` dirties the tree by stamping versions.
(`pnpm pack` has no `--ignore-scripts`, only `pnpm publish` does.)
Three small scripts carry the parts that are easy to get wrong, all runnable
locally:
- **`scripts/verify-pack.mjs`** — packs every publishable package and fails if a
`workspace:` range survived into the tarball. Runs between build and publish
in both workflows. Turns "npm would have shipped a broken manifest" from an
install-time surprise into a red run.
- **`scripts/publish-order.mjs`** — topologically sorts the publishable packages.
`packages/*/` alphabetically puts `nopy` ahead of the `nopy-cube` it depends
on; the snapshot workflow now iterates this instead.
- **`scripts/linked-deps.mjs`** — lists a package's workspace links as
`<name> <version>`, resolved by package name rather than by directory.
`release.yml` uses it to refuse a release whose linked dependency is not on
npmjs yet, which is the one mistake that cannot be taken back after 72 hours.
`publish-snapshot.yml` also became two passes over the packages: stamp every
version first, then publish. `pnpm publish` substitutes the version the linked
package declares *at pack time*, so `nopy-cube` has to be carrying its snapshot
version before `nopy` is packed.
Still unverified: none of this has run against the Gitea registry. Worth a
throwaway version before the first real release. throwaway version before the first real release.
### Also: the resolve hook ### Also: the resolve hook — built
Independent of the split, and worth building anyway — it retires the Independent of the split, and worth building anyway — it retires the
`ERR_MODULE_NOT_FOUND` gotcha CLAUDE.md documents for the local `cubes/` tree, `ERR_MODULE_NOT_FOUND` gotcha CLAUDE.md documents for the local `cubes/` tree,
@@ -414,33 +462,48 @@ Falling back for `zod` hands local cubes the *CLI's* zod instance, so no
duplication arises there. Bundles are the case that duplicates it, and Phase 0.4 duplication arises there. Bundles are the case that duplicates it, and Phase 0.4
is what makes that safe. is what makes that safe.
New `packages/nopy/src/nopy.resolve-hook.mjs`, registered once from `loadCubes()` `packages/nopy/src/cubes/resolve-hook.mjs`, registered once from `loadCubes()`
before the first `import(manifestPath)`: before the first `import(manifestPath)`:
```ts ```ts
module.register('./nopy.resolve-hook.mjs', import.meta.url, { module.register('./resolve-hook.mjs', import.meta.url, { data: { from: import.meta.url } });
data: { fallback: import.meta.resolve('./index.js') },
});
``` ```
The hook tries `next(specifier, ctx)` **first** and only falls back to the `from` is a URL inside the running CLI's own package; the hook thread builds a
running CLI's own copy on failure. That ordering matters: a consumer that has its `createRequire` from it and resolves the fallbacks out of the CLI's own
own `@bitsquare/nopy` installed keeps using it, so the hook never silently dependencies.
introduces version skew.
Constraints: The hook tries `next(specifier, ctx)` **first** and only falls back on failure.
That ordering matters: a consumer that has its own copy installed keeps using
it, so the hook never silently introduces version skew. There is a test for
exactly that — a stub `zod` beside the cube wins over the CLI's real one.
- `module.register()` is process-global and cannot be undone. Install it once, Constraints, as built:
behind a module-level guard.
- `module.register()` is process-global and cannot be undone. Installed once,
behind a module-level guard, and wrapped in a `try` — the hook is a
convenience, so a registration failure must not abort a run.
- The hook file runs on a separate thread; the `data` payload must be - The hook file runs on a separate thread; the `data` payload must be
structured-cloneable (a string URL is). structured-cloneable (a string URL is).
- The `.mjs` must ship in `dist` and be listed in `files` — it already is, via - The `.mjs` has to reach `dist`, and `tsc` does not copy it: nopy's `build` is
the `dist` entry. now `tsc && cp src/cubes/*.mjs dist/cubes/`. `files` already covers it via the
- It resolves `@bitsquare/nopy` and `zod`, not `@bitsquare/nopy-cube`. Bundles `dist` entry.
never depend on the hook; only the in-repo `cubes/` tree and hand-written local - It covers **three** specifiers, not the two the plan named: `zod`,
cubes do. `@bitsquare/nopy`, and `@bitsquare/nopy-cube` — a hand-written local cube is
as entitled to the new authoring package as to the old one. Subpaths count
(`@bitsquare/nopy/package.json`), anything else stays a hard failure.
## Phase 5 — proof of concept: `packages/cubes-core` **The tests have to spawn a real `node`.** Written inside the vitest worker they
pass whether or not the hook is installed: vite resolves the dynamic import
itself and finds `zod` from the project root. `tests/cubes.resolve-hook.test.ts`
therefore runs each case in a child process, and the first case asserts the
*failure* without the hook so the rest cannot silently stop proving anything.
**Verified end to end.** From a plain `node` at the repo root, with nothing
linked, the built loader reads all 22 cubes under `cubes/` with zero errors. The
`ERR_MODULE_NOT_FOUND` gotcha in `CLAUDE.md` is retired.
## Phase 5 — proof of concept: `packages/cubes-core` — **done**
Depends on Phase 4 shipping first — the bundle cannot declare Depends on Phase 4 shipping first — the bundle cannot declare
`@bitsquare/nopy-cube` as a dependency until it exists, and the publish-lane fix `@bitsquare/nopy-cube` as a dependency until it exists, and the publish-lane fix
@@ -478,19 +541,50 @@ has to be in place before either package is published.
`nopy-cube` references from Phase 4 are separate.) `nopy-cube` references from Phase 4 are separate.)
8. Biome already lints `cubes/**/*.mjs` from the root; only the path changes. 8. Biome already lints `cubes/**/*.mjs` from the root; only the path changes.
### Verifying the PoC ### What differed from the plan
- **In-workspace:** `pnpm --filter @bitsquare/nopy run nopy -P` from the repo - **Step 2's optional migration was done.** All 22 manifests now import
root lists `net:tailscale`, `apt:install`, … and prints deploy commands whose `{ Manifest }` from `@bitsquare/nopy-cube`, not `{ cubes }` from
`--chdir` points into `node_modules/@bitsquare/cubes-core/cubes/…`. `@bitsquare/nopy`. Optional for correctness, but it is the only version of the
- **Out-of-workspace (the real test):** `npm pack` the bundle, install the PoC that proves anything: leaving the old import in place would have resolved
tarball into a throwaway directory with a `.nopyrc.json` naming it, install through the CLI that happens to sit in the same tree.
`nopy` *globally*, and run `nopy -P`. This is what actually exercises Phase 4 — - **`uniqid` had to move too.** Two manifests use it (`admin:hostname` bare,
a manifest resolving its import from a `node_modules` tree that has no `user:add` via `cubes.uniqid`), so `src/cubes/utils.ts` and its test went to
`@bitsquare/nopy` in it. Check the installed tarball's `package.json` really `nopy-cube` alongside `types.ts`, and `uniqid` joined the authoring barrel.
carries a concrete `@bitsquare/nopy-cube` range and not `workspace:*`. Otherwise one migrated manifest would still have been importing the CLI.
- **`files` needs a log exclusion.** Cubes that have been run leave a gitignored
`pyinfra-debug.log` next to `deploy.py`; gitignore does not filter an npm
tarball. `"files": ["cubes", "!cubes/**/*.log", …]` does. Verified: 22
manifests, 22 deploy scripts, 0 logs in the packed artefact.
- **`verify-pack.mjs` picks the bundle up for free** — it walks every non-private
`packages/*`, so `cubes-core`'s `workspace:*` edge is checked like nopy's.
## Phase 6 — documentation ### Verifying the PoC — done
- **In-workspace:** the built loader, run from the repo root against the new
root `.nopyrc.json`, reads 22 cubes with 0 errors and reports
`source: { type: 'package', packageName: '@bitsquare/cubes-core', dir:
'…/node_modules/@bitsquare/cubes-core/cubes' }` — the pnpm symlink path, not a
plain directory.
- **Out-of-workspace (the real test):** `pnpm pack` for `nopy-cube`, `nopy` and
`cubes-core`, then **`npm install`** of all three tarballs into a throwaway
directory with a `.nopyrc.json` naming only the bundle. npm is the strict test
here — it does not understand `workspace:`, so a leaked range fails the install
outright. It installed clean, and the installed
`@bitsquare/nopy/package.json` carries `"@bitsquare/nopy-cube":
"1.0.0-alpha0"`. `nopy install -l session.json -P -D` then resolved
`apt:essentials` and printed a `--chdir` into
`node_modules/@bitsquare/cubes-core/cubes/apt/essentials`. Since the loader
aborts on any manifest error and this run did not, all 22 manifests imported
`@bitsquare/nopy-cube` and `zod` successfully from a tree containing no
workspace links.
Note for anyone repeating this: `-P` on its own is interactive, and a replay
still prompts for anything a manifest declares in `secrets` (they are never
persisted to a session) — `net:tailscale` will sit there waiting. Use a
session file with a cube that has no secrets, or answer the prompt.
## Phase 6 — documentation — **done**
- `CLAUDE.md`: the repo table gains two rows (`packages/nopy-cube`, - `CLAUDE.md`: the repo table gains two rows (`packages/nopy-cube`,
`packages/cubes-core`) and loses the `cubes/` one; "The two packages do not `packages/cubes-core`) and loses the `cubes/` one; "The two packages do not
@@ -503,6 +597,21 @@ has to be in place before either package is published.
plus the ordering constraint — `nopy-cube` releases before anything that plus the ordering constraint — `nopy-cube` releases before anything that
depends on it. depends on it.
Beyond the list: `CLAUDE.md` also needed the `typecheck` command corrected
(`tsc --build`, not `--noEmit` — see Phase 4), a note on the vitest source alias
and the coverage exclusion, and three entries under *Known drift*. `README.PUBLISH.md`
absorbed the whole publish-lane rework, not just the tag prefixes: `pnpm publish`
over `npm publish` and why, the two-pass version stamping, `verify-pack.mjs`,
`publish-order.mjs`, `linked-deps.mjs`, and a local rehearsal recipe that uses
**npm** to install the tarballs precisely because npm is the one that rejects a
leaked `workspace:` range.
One workflow change came out of writing this up: `ci.yml` now runs
`verify-pack.mjs` too. It was only in the two publish workflows, which means a
leaked range would have failed the release rather than the pull request that
introduced it — the wrong end of the process for a mistake that is free to catch
early.
## Testing ## Testing
The coverage gate (85 % branches/functions, 80 % lines/statements, per package) The coverage gate (85 % branches/functions, 80 % lines/statements, per package)
+26
View File
@@ -54,3 +54,29 @@ This document tracks the major refactoring of the `nopy` package.
- `VariableAssignment` offers every schema key, not only the ones carrying a default, and shows the value the run would actually use as the initial. - `VariableAssignment` offers every schema key, not only the ones carrying a default, and shows the value the run would actually use as the initial.
- **Proposed Solution**: (Done) - **Proposed Solution**: (Done)
### 6. Make variable assignment a first-class concept
- **Status**: ✅ Completed
- **Goal**: Give a variable an identity and a provenance, instead of inferring both from which bag it happened to sit in.
- **Rationale**: Item 5 left precedence encoded as the field order of an object literal inside `Variables.get()``defaults`, then `global`, then `prompts`, then `params`. Nothing named the ranking, nothing could be asked where a value came from, and a replay had to be smuggled into the `prompts` bag because there was no origin that meant "recorded". Every question that followed — what should a session record, which values are safe to print — needed provenance to answer.
- **Context**:
- `Assignment { value, origin }` and an `Origin` ranked `default(0) < env(1) < session(2) < prompt(3) < param(4)`. Precedence is now data, not the order lines appear in.
- `Variable` is a class over an assignment list. `assignments` is the true history, newest first and never reordered; `ordered` is a *stable* sort of it by origin rank, and `value`/`origin` read the head of that. Stability is what makes the two views coexist: same-origin ties keep the newest in front while the value it displaced stays visible.
- The `global` bag is gone. Config `env` is seeded per cube as a real assignment at origin `env`, so `variables.get('global')` — a cube id that was never a cube — is no longer a thing.
- Replay assigns at origin `session`, which outranks `env` and `default` on its own. The `prompts`-bag workaround is deleted.
- A session records `Variables.persistable()` — every effective value, not just prompted ones. A `-D` run used to record nothing and replay by re-deriving from whatever the defaults said at replay time.
- **Trade-off accepted**: recorded values now outrank the current `.nopyrc.json` `env` and the current schema defaults, so editing either no longer leaks into an existing session's replay. That is the point of a snapshot, but it does mean picking up a new default requires re-recording.
- **Proposed Solution**: (Done)
### 7. Manifest-declared secrets
- **Status**: ✅ Completed
- **Goal**: Let a manifest say which schema keys hold sensitive values, and act on it.
- **Rationale**: Item 6 made sessions record everything, which forced the question of what must *not* be recorded. The codebase already had an answer of sorts — `outputExecutionPlan` masked any variable whose name contained "password" — that missed `TOKEN`, `PSK` and `AUTH_KEY`, and was defeated anyway by the unmasked command printed one line above it.
- **Context**:
- `Manifest.secrets?: string[]`, validated at load: an entry that is not a key of `schema` is a manifest error and aborts the run, so a typo cannot silently leave a value unprotected.
- Deliberately a plain array, not zod metadata. `.meta()` and `.describe()` store into `z.globalRegistry`, which is per-copy — a manifest built by a different zod copy would look up empty. Fail-open is fine for a missing prompt label and unacceptable for a secret marker.
- `maskCommand()` replaces declared `--data` values and the SSH `--password` in the command string itself, and is wired into `--print-only`, the dry-run plan and the debug log. The `nopy` logger runs at `lowestLevel: 'debug'`, so that last one was printing credentials on every run.
- Secrets are excluded from `persistable()`, so a replay has a gap where one used to be. `fillSessionGaps` prompts for `requiredKeys() secrets`; under `-D` it fails naming them, consistent with item 5's fail-fast.
- **Scope limit**: `secrets` keeps a value out of what nopy writes. The value is still on pyinfra's command line (visible in `ps`), still echoed by the variable form, and a `.default()` is still plain text in the manifest. Documented rather than fixed — the first is inherent to pyinfra's interface.
- **Bug fixed along the way**: `cubes/user/add` generated a random password as its schema `.default()`. Because the key had a default it was never in `requiredKeys()`, and because a generated default is re-evaluated on every read, an unattended run created an account with a credential nobody had seen and a replay created a different one again. It is now the literal `changeme`.
- **Proposed Solution**: (Done)
+15 -1
View File
@@ -294,8 +294,9 @@ export default {
2. **Document your cubes** - Add comments explaining what each cube does 2. **Document your cubes** - Add comments explaining what each cube does
3. **Use environment variables** - Make sessions reusable across environments 3. **Use environment variables** - Make sessions reusable across environments
4. **Extract common config** - Share configuration across multiple sessions 4. **Extract common config** - Share configuration across multiple sessions
5. **Version control** - Both formats work well with git 5. **Version control** - Both formats work well with git, but a recorded session holds every value its run used; read one before committing it
6. **Validate at runtime** - The loader validates the structure regardless of format 6. **Validate at runtime** - The loader validates the structure regardless of format
7. **Leave secrets out** - Declare them in the manifest instead, and let the replay ask
## Session Schema ## Session Schema
@@ -321,3 +322,16 @@ interface AuthSession {
username?: string; username?: string;
} }
``` ```
A session nopy *writes* holds, per cube, every value that cube ran with — what
was typed, what came from `.nopyrc.json`, what a dependency supplied, and what
fell through to the schema's `.default()`. Two things are deliberately absent and
are asked for again on replay: the SSH password, and any key the cube's manifest
listed under `secrets`.
A session you write by hand is under no such obligation — `variables` may hold as
few keys as you like, and anything missing resolves the usual way. Note that a
key declared a secret is prompted for whether or not the session carries a value:
writing one in only pre-fills the prompt, it does not skip it. The variable form
shows what it is editing, so a secret you put in a session file appears on screen
as well as on disk.
+2 -1
View File
@@ -43,7 +43,7 @@
}, },
"scripts": { "scripts": {
"clean": "rm -rf dist .tsbuildinfo", "clean": "rm -rf dist .tsbuildinfo",
"build": "tsc", "build": "tsc && cp src/cubes/*.mjs dist/cubes/",
"prepack": "pnpm run build", "prepack": "pnpm run build",
"link:local": "pnpm run build && npm link", "link:local": "pnpm run build && npm link",
"nopy": "tsx src/nopy.cli.ts", "nopy": "tsx src/nopy.cli.ts",
@@ -54,6 +54,7 @@
"test:watch": "vitest" "test:watch": "vitest"
}, },
"dependencies": { "dependencies": {
"@bitsquare/nopy-cube": "workspace:*",
"@logtape/logtape": "^2.2.4", "@logtape/logtape": "^2.2.4",
"commander": "^15.0.0", "commander": "^15.0.0",
"enquirer": "^2.4.1", "enquirer": "^2.4.1",
+59 -10
View File
@@ -3,13 +3,13 @@
* @module cubes/dependencies * @module cubes/dependencies
*/ */
import type { Cube, CubeVariables, HookContext } from '@bitsquare/nopy-cube';
import { getLogger } from '@logtape/logtape'; import { getLogger } from '@logtape/logtape';
import type { Variables } from '../nopy.common.js'; import type { Variables } from '../nopy.common.js';
import type { NopyConfig } from '../nopy.config.js'; import type { NopyConfig } from '../nopy.config.js';
import type { DeployCall } from '../nopy.executor.js'; import type { DeployCall } from '../nopy.executor.js';
import { VariableAssignment } from '../nopy.prompts.js'; import { VariableAssignment } from '../nopy.prompts.js';
import type { CubeSession, NopySession } from '../nopy.session.js'; import type { CubeSession, NopySession } from '../nopy.session.js';
import type { Cube, CubeVariables, HookContext } from './types.js';
const log = getLogger(['nopy', 'resolution']); const log = getLogger(['nopy', 'resolution']);
@@ -38,6 +38,12 @@ export class BuildContext {
} = {} } = {}
) {} ) {}
/** Required schema keys that nothing has supplied a value for. */
private missingRequired(cube: Cube): string[] {
const resolved = this.variables.get(cube.id);
return cube.requiredKeys().filter((key) => resolved[key] === undefined);
}
/** /**
* Fails a non-interactive run that cannot fill a required variable. * Fails a non-interactive run that cannot fill a required variable.
* *
@@ -45,8 +51,7 @@ export class BuildContext {
* `--data`, and the deploy script would read `None` off `host.data`. * `--data`, and the deploy script would read `None` off `host.data`.
*/ */
private assertVariablesComplete(cube: Cube): void { private assertVariablesComplete(cube: Cube): void {
const resolved = this.variables.get(cube.id); const missing = this.missingRequired(cube);
const missing = cube.requiredKeys().filter((key) => resolved[key] === undefined);
if (missing.length === 0) return; if (missing.length === 0) return;
const [one, them] = const [one, them] =
@@ -58,6 +63,43 @@ export class BuildContext {
); );
} }
/**
* Asks for the variables a replay cannot supply on its own.
*
* Two kinds. Required keys can be absent because the session predates them or
* was recorded by a `--use-defaults` run. Secrets are absent by design: they
* are never written to a session, so replaying without asking would deploy a
* cube with the key missing — or, for a secret carrying a default, with a
* value silently different from the run being replayed.
*
* Secrets are asked for even when a default did fill them in, which is why
* this cannot key off "has no value": the whole point is that the recorded
* answer is gone and only the user knows what it was.
*/
private async fillSessionGaps(cube: Cube): Promise<void> {
const gaps = [...new Set([...this.missingRequired(cube), ...cube.secrets])];
if (gaps.length === 0) return;
if (this.options.useDefaults) {
throw new Error(
`Cube "${cube.id}" cannot be replayed with --use-defaults: ${gaps.join(', ')} ` +
'would have to be entered. Secrets are never recorded in a session. ' +
'Replay without --use-defaults, or set the values under "env" in .nopyrc.json.'
);
}
log.debug('Filling session gaps', { cubeId: cube.id, gaps });
await VariableAssignment(cube, this.variables, { keys: gaps });
// A cancelled form leaves the run short of a value it cannot invent.
const stillMissing = this.missingRequired(cube);
if (stillMissing.length > 0) {
throw new Error(
`Cube "${cube.id}" is missing ${stillMissing.join(', ')} and cannot be deployed.`
);
}
}
/** /**
* Resolves a cube, its dependencies, and hooks recursively * Resolves a cube, its dependencies, and hooks recursively
*/ */
@@ -73,20 +115,22 @@ export class BuildContext {
log.debug('Resolving cube', { cubeId, host }); log.debug('Resolving cube', { cubeId, host });
// 1. Assign overrides and defaults // 1. Declare secrets, then assign overrides and defaults. Declaring first
// means even the config `env` seeded on the cube's first assignment is
// already marked, so nothing reaches a session or a log unredacted.
this.variables.declareSecrets(cubeId, cube.secrets);
if (Object.keys(overrides).length > 0) { if (Object.keys(overrides).length > 0) {
this.variables.assign(cubeId, 'params', overrides); this.variables.assign(cubeId, 'param', overrides);
} }
this.variables.assign(cubeId, 'defaults', cube.getDefaults()); this.variables.assign(cubeId, 'default', cube.getDefaults());
// 2. Variable collection // 2. Variable collection
if (this.options.isSessionReplay) { if (this.options.isSessionReplay) {
// Recorded answers go back into the scope they came from, so a replay
// reproduces them even when `env` sets the same key to something else.
const sessionCube = this.session.cubes.find((c) => c.key === cubeId); const sessionCube = this.session.cubes.find((c) => c.key === cubeId);
if (sessionCube) { if (sessionCube) {
this.variables.assign(cubeId, 'prompts', sessionCube.variables); this.variables.assign(cubeId, 'session', sessionCube.variables);
} }
await this.fillSessionGaps(cube);
} else if (this.options.useDefaults) { } else if (this.options.useDefaults) {
log.debug('Skipping prompts, using defaults', { cubeId }); log.debug('Skipping prompts, using defaults', { cubeId });
this.assertVariablesComplete(cube); this.assertVariablesComplete(cube);
@@ -155,13 +199,18 @@ export class BuildContext {
cwd: cube.dir, cwd: cube.dir,
command, command,
env: cubeVars, env: cubeVars,
secrets: cube.secrets,
dependencies: [], dependencies: [],
}); });
if (!this.cubeSessions.some((s) => s.key === cubeId)) { if (!this.cubeSessions.some((s) => s.key === cubeId)) {
// Every value the run settled on, not just the prompted ones — otherwise a
// `--use-defaults` run records nothing and replaying it re-derives from
// whatever the defaults and `env` happen to say now. Secrets are the one
// exclusion; a replay asks for those again.
this.cubeSessions.push({ this.cubeSessions.push({
key: cubeId, key: cubeId,
variables: this.variables.get(cubeId, 'prompts'), variables: this.variables.persistable(cubeId),
}); });
} }

Some files were not shown because too many files have changed in this diff Show More