[fix] default parameter run records parameters in session for replay[fix] remove default parameters for several cubes
This commit is contained in:
@@ -81,6 +81,13 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
done
|
||||
|
||||
- name: Verify the packed manifests
|
||||
# `workspace:*` is mandatory in the manifests but meaningless to npm, so
|
||||
# a range that survives into a tarball is an install failure for every
|
||||
# consumer. The publish workflows run this too; running it here is what
|
||||
# puts the failure on the pull request instead of on the release.
|
||||
run: node scripts/verify-pack.mjs
|
||||
|
||||
- name: Upload coverage reports
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Every commit that lands on `main` publishes a prerelease of both packages to
|
||||
# the Gitea npm registry under the `main` dist-tag:
|
||||
# Every commit that lands on `main` publishes a prerelease of every publishable
|
||||
# package to the Gitea npm registry under the `main` dist-tag:
|
||||
#
|
||||
# pnpm add @bitsquare/nopy@main
|
||||
#
|
||||
@@ -78,6 +78,11 @@ jobs:
|
||||
# Explicit, so the publish step can skip lifecycle scripts entirely.
|
||||
run: pnpm run build
|
||||
|
||||
- name: Verify the packed manifests
|
||||
# Packages link to each other with `workspace:*`, which npm cannot
|
||||
# install. Proves on the tarball that pack rewrote it.
|
||||
run: node scripts/verify-pack.mjs
|
||||
|
||||
- name: Authenticate against the Gitea registry
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -97,23 +102,35 @@ jobs:
|
||||
export npm_config_userconfig="$NPMRC"
|
||||
: "${GITHUB_STEP_SUMMARY:=/dev/null}"
|
||||
short_sha=$(git rev-parse --short=7 HEAD)
|
||||
# Dependencies first, so the registry never briefly holds a package
|
||||
# whose dependency has not landed yet.
|
||||
dirs=$(node scripts/publish-order.mjs)
|
||||
|
||||
for dir in packages/*/; do
|
||||
name=$(node -p "require('./${dir}package.json').name")
|
||||
base=$(node -p "require('./${dir}package.json').version")
|
||||
# Pass 1: stamp every manifest before anything is packed. `pnpm
|
||||
# publish` substitutes `workspace:*` with the version the linked
|
||||
# package declares at pack time, so nopy-cube has to be carrying its
|
||||
# snapshot version by the time nopy is packed.
|
||||
for dir in $dirs; do
|
||||
base=$(node -p "require('./${dir}/package.json').version")
|
||||
# `g` prefix keeps the identifier a valid semver one even when the
|
||||
# abbreviated sha happens to be all digits.
|
||||
version="${base}-main.${{ github.run_number }}.g${short_sha}"
|
||||
(cd "$dir" && npm pkg set "version=${version}")
|
||||
done
|
||||
|
||||
# Pass 2: publish.
|
||||
for dir in $dirs; do
|
||||
name=$(node -p "require('./${dir}/package.json').name")
|
||||
version=$(node -p "require('./${dir}/package.json').version")
|
||||
|
||||
echo "::group::${name}@${version}"
|
||||
if npm view "${name}@${version}" version --registry "$REGISTRY" >/dev/null 2>&1; then
|
||||
echo "Already published — skipping (this is a re-run of the same workflow)."
|
||||
else
|
||||
(
|
||||
cd "$dir"
|
||||
npm pkg set "version=${version}"
|
||||
npm publish --ignore-scripts --tag main --registry "$REGISTRY"
|
||||
)
|
||||
# pnpm, not npm: npm ships `workspace:*` verbatim and the install
|
||||
# then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because
|
||||
# stamping the versions above left the tree dirty.
|
||||
(cd "$dir" && pnpm publish --ignore-scripts --no-git-checks --tag main --registry "$REGISTRY")
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
# Tag-driven release of a single package.
|
||||
#
|
||||
# git tag nopy-v1.2.0 && git push origin nopy-v1.2.0
|
||||
# git tag nopy-cube-v1.2.0 && git push origin nopy-cube-v1.2.0
|
||||
# git tag keyman-v1.2.0 && git push origin keyman-v1.2.0
|
||||
#
|
||||
# The tag is the source of truth for *which* package ships; package.json is the
|
||||
# source of truth for the version, and the two must agree or the run fails.
|
||||
#
|
||||
# Packages that link to each other release dependency-first — `nopy-cube` before
|
||||
# `nopy` — because the linked version is resolved at pack time. The run refuses
|
||||
# to publish otherwise.
|
||||
# A version with a prerelease part (1.2.0-rc.1) publishes under `next` instead
|
||||
# of `latest`.
|
||||
#
|
||||
@@ -108,6 +113,31 @@ jobs:
|
||||
- name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Check the linked workspace packages are already released
|
||||
env:
|
||||
NAME: ${{ steps.target.outputs.name }}
|
||||
DIR: ${{ steps.target.outputs.dir }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# `pnpm publish` turns `workspace:*` into the version the linked
|
||||
# package declares at this commit. If that version is not on the
|
||||
# registry yet, the release installs to a broken tree — and npmjs
|
||||
# only lets you unpublish for 72 hours. Release the dependency first:
|
||||
# nopy-cube, then nopy, then any bundle.
|
||||
#
|
||||
# npmjs only: it is the irreversible one, and it needs no credentials
|
||||
# to read, which this step does not have yet.
|
||||
missing=0
|
||||
for spec in $(node scripts/linked-deps.mjs "$DIR" | tr ' ' '@'); do
|
||||
if npm view "$spec" version --registry "$NPMJS_REGISTRY" >/dev/null 2>&1; then
|
||||
echo "${spec} is published"
|
||||
else
|
||||
echo "::error::${NAME} depends on ${spec}, which is not on npmjs. Release it first."
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
exit "$missing"
|
||||
|
||||
- name: Lint
|
||||
run: pnpm run lint:ci
|
||||
|
||||
@@ -121,6 +151,11 @@ jobs:
|
||||
# Explicit, so the publish steps can skip lifecycle scripts entirely.
|
||||
run: pnpm run build
|
||||
|
||||
- name: Verify the packed manifests
|
||||
# Packages link to each other with `workspace:*`, which npm cannot
|
||||
# install. Proves on the tarball that pack rewrote it.
|
||||
run: node scripts/verify-pack.mjs
|
||||
|
||||
- name: Publish to the Gitea registry
|
||||
env:
|
||||
NAME: ${{ steps.target.outputs.name }}
|
||||
@@ -139,7 +174,10 @@ jobs:
|
||||
if npm view "${NAME}@${VERSION}" version --registry "$GITEA_REGISTRY" >/dev/null 2>&1; then
|
||||
echo "${NAME}@${VERSION} is already on Gitea — skipping."
|
||||
else
|
||||
(cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --registry "$GITEA_REGISTRY")
|
||||
# pnpm, not npm: npm ships `workspace:*` verbatim and the install
|
||||
# then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because a
|
||||
# tag build is a detached HEAD.
|
||||
(cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --registry "$GITEA_REGISTRY")
|
||||
fi
|
||||
|
||||
- name: Publish to npmjs
|
||||
@@ -162,7 +200,7 @@ jobs:
|
||||
else
|
||||
# No --provenance: that needs GitHub Actions OIDC, which Gitea has no
|
||||
# equivalent for.
|
||||
(cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY")
|
||||
(cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY")
|
||||
fi
|
||||
|
||||
- name: Remove the registry credentials
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"hosts": [],
|
||||
"cubeDirs": ["./cubes"],
|
||||
"cubeDirs": [],
|
||||
"cubePackages": ["@bitsquare/cubes-core"],
|
||||
"env": {},
|
||||
"log": {
|
||||
"verbosity": "info",
|
||||
|
||||
@@ -4,26 +4,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
|
||||
## What this repo is
|
||||
|
||||
A pnpm workspace holding two independently published CLIs plus the pyinfra
|
||||
deployment units one of them runs:
|
||||
A pnpm workspace holding two independently published CLIs, the authoring package
|
||||
their deployment units are written against, and one bundle of those units:
|
||||
|
||||
| Path | Package | Binary | Role |
|
||||
| ----------------- | ------------------- | -------- | -------------------------------------------------------- |
|
||||
| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution |
|
||||
| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` |
|
||||
| `cubes/` | — | — | the deployment units `nopy` runs (not published) |
|
||||
| Path | Package | Binary | Role |
|
||||
| --------------------- | ---------------------- | -------- | -------------------------------------------------------- |
|
||||
| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution |
|
||||
| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` |
|
||||
| `packages/nopy-cube` | `@bitsquare/nopy-cube` | — | the authoring surface a `manifest.mjs` imports |
|
||||
| `packages/cubes-core` | `@bitsquare/cubes-core`| — | the core cube bundle (22 cubes), no TypeScript |
|
||||
|
||||
The root package is private; only `packages/*` ship. The two packages do not
|
||||
depend on each other.
|
||||
The root package is private; everything under `packages/` ships. `keyman` stands
|
||||
alone, but `nopy` and `cubes-core` both depend on `nopy-cube` (`workspace:*`), so
|
||||
publish order matters — see *Releasing*.
|
||||
|
||||
`cubes-core` is consumed the way a third party would consume it: the root
|
||||
`.nopyrc.json` names it in `cubePackages`, and the loader reads it out of
|
||||
`node_modules`. There is no `cubes/` directory at the repo root any more.
|
||||
|
||||
## Commands
|
||||
|
||||
```sh
|
||||
pnpm install # also installs the git hooks via simple-git-hooks
|
||||
pnpm run build # tsc --build across both packages (project references)
|
||||
pnpm run typecheck # tsc --build --noEmit
|
||||
pnpm run build # tsc --build across the TS packages (project references)
|
||||
pnpm run typecheck # tsc --build (see below — it really does emit)
|
||||
pnpm run lint # biome check . (lint:fix / lint:ci variants)
|
||||
pnpm test # vitest run, both packages
|
||||
pnpm test # vitest run, every package with tests
|
||||
pnpm run test:coverage # vitest with the coverage gate
|
||||
pnpm run coverage:summary # renders the last coverage run as a Markdown table
|
||||
```
|
||||
@@ -41,6 +47,12 @@ pnpm --filter @bitsquare/keyman run keyman
|
||||
`typescript` is the 7.x native compiler, so `tsc` *is* the fast one — there is no
|
||||
separate `tsgo` binary.
|
||||
|
||||
`typecheck` is plain `tsc --build`, not `--noEmit`. Once a project has
|
||||
`references`, `--noEmit` is rejected outright (TS6310: *referenced project may
|
||||
not disable emit*) — a composite project has to emit the declarations its
|
||||
dependents read. So the typecheck writes `dist` as a side effect; it is
|
||||
gitignored, and the upside is that the gate now also proves the build works.
|
||||
|
||||
## Verification gate
|
||||
|
||||
`lint:ci` → `typecheck` → `test:coverage` is one gate, run in three places: the
|
||||
@@ -55,7 +67,14 @@ locally and on the runner. Barrel files (`src/index.ts`, `src/cubes/index.ts`,
|
||||
`src/nopy.cubes.ts`) and the Commander argv wiring (`src/*.cli.ts`) are excluded;
|
||||
adding logic to those files means moving it somewhere covered.
|
||||
|
||||
Both packages set `pool: 'forks'` because tests use `process.chdir()` — most
|
||||
nopy's vitest config aliases `@bitsquare/nopy-cube` to that package's **source**,
|
||||
not to the workspace link (which points at a `dist` that only exists after a
|
||||
build), so the gate does not depend on build ordering and can never run against a
|
||||
stale artefact. The same config excludes `**/nopy-cube/**` from coverage — without
|
||||
it nopy's numbers absorb another package's files. `cubes-core` has no tests of its
|
||||
own; the loader tests in nopy cover the contract it implements.
|
||||
|
||||
The three TS packages set `pool: 'forks'` because tests use `process.chdir()` — most
|
||||
loader/config tests build a throwaway tree under `os.tmpdir()` and chdir into it,
|
||||
since discovery is driven entirely by the working directory.
|
||||
|
||||
@@ -68,22 +87,43 @@ One pass per invocation, `nopy.main.ts` orchestrating:
|
||||
Per-property strategy comes from the child's `resolution` block (`merge` is
|
||||
the default: arrays concatenate and dedupe, objects deep-merge; `override`
|
||||
replaces). Only properties listed in `PATH_PROPERTIES` (`cubeDirs`) get
|
||||
relative paths resolved against their own config file's directory. **Throws**
|
||||
relative paths resolved against their own config file's directory;
|
||||
`cubePackages` needs the same origin for a different reason, so each entry is
|
||||
normalised into a `CubePackageRef {spec, from}` — `from` is the directory of
|
||||
the config that named it, which is where the package gets resolved from.
|
||||
**Throws**
|
||||
if no config file exists anywhere — which is why `nopy.cli.ts` calls it lazily
|
||||
inside the action, so `--help`/`--version` work outside a project.
|
||||
2. **`cubes/loader.ts`** — `findCubeDirectories()` unions `config.cubeDirs` with
|
||||
every ancestor directory holding a `.npcubes` marker, then scans each
|
||||
recursively (skipping dotted dirs and `node_modules`). A directory is a cube
|
||||
2. **`cubes/packages.ts`** — `resolveCubePackages()` turns each `CubePackageRef`
|
||||
into a package root plus the directories its `nopy.cubes` field declares.
|
||||
Resolution goes through `createRequire(...).resolve.paths()` + `existsSync`,
|
||||
deliberately bypassing the `exports` map: a bundle ships directories and has
|
||||
no entry point to declare. `existsSync` also follows the symlink pnpm plants
|
||||
at `node_modules/<name>`, which a `readdir` scan skips outright (it reports
|
||||
`isSymbolicLink()`, not `isDirectory()`). A missing package, an unreadable
|
||||
manifest, a missing `nopy.cubes`, a directory that does not exist, and an
|
||||
entry pointing outside the package root are all errors, never silent skips.
|
||||
Duplicate refs are deduped here, last-wins, because `mergeValue` only dedupes
|
||||
arrays of primitives and these are objects.
|
||||
3. **`cubes/loader.ts`** — `findCubeRoots()` unions `config.cubeDirs`, the
|
||||
directories from `cubePackages`, and every ancestor directory holding a
|
||||
`.npcubes` marker, then scans each recursively (skipping dotted dirs and
|
||||
`node_modules`). A directory is a cube
|
||||
when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a
|
||||
deploy script (`deploy.py` or `*.deploy.py`); manifests are loaded by dynamic
|
||||
`import()`. Cube id = `manifest.id` → a `[id]` prefix in `manifest.name` →
|
||||
the directory basename. Ids are flat and need not mirror the path
|
||||
(`cubes/network/tailscale` declares `net:tailscale`). Duplicate ids and bad
|
||||
manifests become entries in `errors`, which aborts the run.
|
||||
3. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and
|
||||
(`cubes/network/tailscale` declares `net:tailscale`), and they are claimed
|
||||
**globally**, not per source: a duplicate is a hard error naming every
|
||||
claimant, with no precedence rule and no shadowing. Each cube carries a
|
||||
`source` — `{type: 'dir', dir}` or `{type: 'package', packageName, dir}` —
|
||||
which is what makes that error legible when the collision is between a local
|
||||
tree and an installed bundle. Duplicate ids and bad manifests become entries
|
||||
in `errors`, which aborts the run.
|
||||
4. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and
|
||||
normalises all three into a `WorkflowResult`. Replays never re-prompt except
|
||||
for passwords (never persisted) and a missing host.
|
||||
4. **`cubes/dependencies.ts` → `BuildContext.resolveCube()`** — the core.
|
||||
5. **`cubes/dependencies.ts` → `BuildContext.resolveCube()`** — the core.
|
||||
Recursive, per (cube, host): assign params and schema defaults → collect
|
||||
variables (prompt, or read them back from the session on replay) → run
|
||||
`before` hooks → resolve `manifest.dependencies(vars)` (dynamic: it receives
|
||||
@@ -92,33 +132,71 @@ One pass per invocation, `nopy.main.ts` orchestrating:
|
||||
`${cubeId}:${host}` set makes emission idempotent. Hooks get a `HookContext`
|
||||
whose `exec(id, vars)` re-enters `resolveCube`, so a hook can pull in a cube
|
||||
that is not a declared dependency.
|
||||
5. **`nopy.executor.ts`** — runs the built `pyinfra <host> -y --data K=V ... --chdir <cubeDir> <script>`
|
||||
6. **`nopy.executor.ts`** — runs the built `pyinfra <host> -y --data K=V ... --chdir <cubeDir> <script>`
|
||||
commands through execa with inherited stdio, sequentially, stopping at the
|
||||
first failure unless `continueOnError`.
|
||||
|
||||
### Variables
|
||||
|
||||
`Variables` (`nopy.common.ts`) keeps three per-cube scopes plus one global bag.
|
||||
`get(id)` merges them lowest-to-highest: `global` (from config `env`) →
|
||||
`defaults` (Zod `.default()`, and recorded session values on replay) → `prompts`
|
||||
(what the user typed) → `params` (values passed by a dependency spec or hook).
|
||||
`Variables` (`nopy.common.ts`) holds one `Variable` per (cube, key). A `Variable`
|
||||
is a list of `Assignment {value, origin}`, and precedence is the `Origin` rank:
|
||||
`default(0) < env(1) < session(2) < prompt(3) < param(4)`. There are no scope
|
||||
bags — config `env` is seeded per cube as a real assignment, so the old
|
||||
`get('global')` (a cube id that was never a cube) is gone, and a replay assigns at
|
||||
`session` instead of being smuggled into the prompts bag.
|
||||
|
||||
`assignments` is the true history, newest first, never reordered. `ordered` is a
|
||||
**stable** sort of it by rank; `value` / `origin` read its head. The stability is
|
||||
load-bearing: it is what makes same-origin ties resolve to the newest while the
|
||||
displaced value stays visible in the trace. The trace is never persisted.
|
||||
|
||||
`get(id)` returns the effective values (→ the pyinfra command line);
|
||||
`persistable(id)` returns the same minus declared secrets (→ session and history).
|
||||
Every schema key is guaranteed present on the pyinfra side; pyinfra parses
|
||||
`--data` values itself, so `"true"` arrives as a bool and numeric strings as ints.
|
||||
|
||||
A manifest's `secrets: string[]` names schema keys holding sensitive values —
|
||||
validated at load (an entry that is not a schema key aborts the run). Secrets are
|
||||
excluded from `persistable()`, re-prompted on replay via `fillSessionGaps`
|
||||
(`requiredKeys() ∪ secrets`), and masked by `maskCommand()` / `maskVariables()`
|
||||
wherever a command is printed. Deliberately a plain array rather than zod
|
||||
metadata: `.meta()` and `.describe()` live in the per-copy `z.globalRegistry`, so
|
||||
a manifest built by a different zod copy would look up empty — fail-open is
|
||||
tolerable for a prompt label and not for a secret marker. See `docs/REFACTORING.md`
|
||||
items 6 and 7.
|
||||
|
||||
### Cube contract
|
||||
|
||||
A cube directory holds `manifest.mjs` + `deploy.py`; anything else in it is
|
||||
ignored by the loader but reachable from the script, which runs with the cube
|
||||
directory as its cwd. Manifests are ESM, import `cubes.Manifest` from
|
||||
`@bitsquare/nopy`, and declare `id`, `name`, a Zod `schema` (each field
|
||||
directory as its cwd. Manifests are ESM, import `Manifest` from
|
||||
`@bitsquare/nopy-cube`, and declare `id`, `name`, a Zod `schema` (each field
|
||||
`.describe()`d — the description is the prompt label — and `.default()`ed), plus
|
||||
optional `dependencies`/`before`/`after`.
|
||||
optional `secrets`/`dependencies`/`before`/`after`.
|
||||
|
||||
**Gotcha:** those manifests resolve `@bitsquare/nopy` through ordinary Node
|
||||
resolution from the manifest's own directory. Nothing in this repo links the
|
||||
package into `cubes/` or `packages/nopy/cubes/`, so loading them fails with
|
||||
`ERR_MODULE_NOT_FOUND` until you link it (`pnpm --filter @bitsquare/nopy run
|
||||
link:local`, then `npm link @bitsquare/nopy` where you run from).
|
||||
Import from **`@bitsquare/nopy-cube`**, not `@bitsquare/nopy`. The authoring
|
||||
surface is types and a factory, with zod as its only peer — no CLI, no prompts,
|
||||
no process spawning — so a bundle can depend on it without dragging the CLI in.
|
||||
`@bitsquare/nopy` re-exports all of it (`cubes.Manifest`, `cubes.uniqid`, …), so
|
||||
the older form still works; every cube in `packages/cubes-core` has been moved to
|
||||
the new one.
|
||||
|
||||
Manifests are resolved by ordinary Node resolution **from the manifest's own
|
||||
directory**, which used to mean a hand-written local cube failed with
|
||||
`ERR_MODULE_NOT_FOUND` unless you linked the package. `cubes/resolve-hook.mjs`
|
||||
retires that: `loadCubes()` registers a `module.register()` resolve hook that
|
||||
tries normal resolution *first* and only on failure falls back to resolving
|
||||
`@bitsquare/nopy-cube`, `@bitsquare/nopy` and `zod` from the running CLI's own
|
||||
`node_modules`. Ordinary-resolution-first is the load-bearing part — a cube that
|
||||
ships its own zod keeps it. The hook is a convenience, never load-bearing:
|
||||
registration is wrapped in a `try`, and a bundle installed properly never reaches
|
||||
it. `dist/cubes/*.mjs` is copied by the build, not compiled — hence
|
||||
`"build": "tsc && cp src/cubes/*.mjs dist/cubes/"`.
|
||||
|
||||
Its tests must spawn a real `node` child process. Written inside the vitest
|
||||
worker they prove nothing: vite resolves the dynamic import itself, so they pass
|
||||
whether or not the hook is installed — verified by commenting the registration
|
||||
out and watching them stay green.
|
||||
|
||||
## keyman architecture
|
||||
|
||||
@@ -134,7 +212,7 @@ the environment beats the config file. Encryption shells out to `age` /
|
||||
|
||||
Tag-driven, one package at a time; see `README.PUBLISH.md`.
|
||||
|
||||
- Push to `main` → `publish-snapshot.yml` publishes both packages to the Gitea
|
||||
- Push to `main` → `publish-snapshot.yml` publishes every package to the Gitea
|
||||
registry as `<version>-main.<run>.g<sha>` under the `main` dist-tag. The
|
||||
version is set on the runner with `npm pkg set` and never committed.
|
||||
- `git tag <dir>-v<version>` (e.g. `nopy-v1.2.0` — the directory under
|
||||
@@ -145,8 +223,42 @@ Tag-driven, one package at a time; see `README.PUBLISH.md`.
|
||||
|
||||
So: bump `packages/<pkg>/package.json`, land it on `main`, then tag that commit.
|
||||
|
||||
Three things the `workspace:*` links added, all of them non-obvious:
|
||||
|
||||
- **`pnpm publish`, never `npm publish`.** `link-workspace-packages` is unset and
|
||||
pnpm 10+ defaults it to `false`, so `workspace:*` is mandatory in the manifests
|
||||
— and npm does not understand it. `npm pack` ships the literal string and the
|
||||
install fails with `EUNSUPPORTEDPROTOCOL`; `pnpm pack`/`pnpm publish` substitute
|
||||
the real version at pack time. Both directions were measured, not assumed.
|
||||
- **`scripts/verify-pack.mjs`** packs every non-private package and fails if any
|
||||
`workspace:` range survived into a tarball. It runs in both publish workflows.
|
||||
Note `pnpm pack` has no `--ignore-scripts` flag, so `prepack` does rebuild —
|
||||
which means the artefact under test is the one publish ships.
|
||||
- **Order.** `packages/*/` sorts `nopy` before `nopy-cube`, which is backwards.
|
||||
`scripts/publish-order.mjs` topologically sorts over the `workspace:` edges;
|
||||
the snapshot workflow stamps *every* version first and only then publishes in
|
||||
that order, because `pnpm publish` reads the linked package's version at pack
|
||||
time. `release.yml` additionally refuses to ship a package whose linked
|
||||
dependency is not yet on npmjs (`scripts/linked-deps.mjs`) — npmjs is the
|
||||
registry you cannot take a mistake back from.
|
||||
|
||||
## Known drift
|
||||
|
||||
`logConfigToFlags()` is exported and tested but nothing feeds its output into the
|
||||
built pyinfra command, so `log.verbosity` / `log.debug` in `.nopyrc.json`
|
||||
currently have no effect. Treat `docs/REFACTORING.md` as a plan, not a record.
|
||||
|
||||
The publish-lane changes above have been verified locally (pack, npm-install of
|
||||
the tarballs into a throwaway tree, run) but have **never run against the Gitea
|
||||
registry**. Burn a throwaway version there before the first real release.
|
||||
|
||||
Nothing checks that a bundle and the CLI reading it are compatible versions;
|
||||
`nopy.engines` was considered and deferred. `docs/CUBE-PACKAGES.md` is where all
|
||||
of this came from and is now a record of what was built, including what differed
|
||||
from the plan.
|
||||
|
||||
`docs/API.md` predates several refactors and still describes `Cube` and
|
||||
`Manifest` as plain interfaces with a `key` field; the code has a `Cube` class
|
||||
keyed on `id`. The `cubePackages` and `CubeSource` sections added for this work
|
||||
are accurate; treat the rest of that file with suspicion. `DOCS-AUDIT.md` tracks
|
||||
the wider drift.
|
||||
|
||||
+78
-11
@@ -16,7 +16,9 @@ state.
|
||||
|
||||
Findings closed since are marked **✅ … fixed** and keep their original text as
|
||||
the record of what was wrong. So far: §1.1 (`--use-defaults`), §2.2
|
||||
(`getDefaults()`), half of §2.1 (precedence), and one bullet of §6.4.
|
||||
(`getDefaults()`), §2.1 (precedence — the second half closed differently than
|
||||
proposed), §4.2 (password on stdout — points 1 and 2 of 3), §4.3 (what a session
|
||||
records), part of §3.5 (dead exports), and one bullet of §6.4.
|
||||
|
||||
---
|
||||
|
||||
@@ -138,14 +140,22 @@ dependency detected".
|
||||
|
||||
## 2. Documented behaviour that differs from the code
|
||||
|
||||
### 2.1 🟡 Variable precedence is wrong in both directions — **partly fixed**
|
||||
### 2.1 ✅ Variable precedence is wrong in both directions — **fixed**
|
||||
|
||||
> **Half resolved.** `env` now outranks the Zod defaults, as documented — this
|
||||
> was a prerequisite for `--use-defaults` being configurable at all. The second
|
||||
> pair was deliberately left: dependency/hook params still outrank prompts. In
|
||||
> practice they do not compete, because `VariableAssignment` leaves out any key
|
||||
> a dependency already supplied, so the operator is never asked about it. The
|
||||
> README now describes that rather than the old claim.
|
||||
> **Resolved, though the second pair closed the opposite way to the README's
|
||||
> original claim.** `env` now outranks the Zod defaults, as documented — that was
|
||||
> a prerequisite for `--use-defaults` being configurable at all.
|
||||
>
|
||||
> Dependency/hook params still outrank prompts, deliberately. They do not compete
|
||||
> in practice: `VariableAssignment` leaves out any key a dependency supplied, so
|
||||
> the operator is never asked about it and there is no typed value to override.
|
||||
> The README documents the real order rather than the old promise.
|
||||
>
|
||||
> The underlying complaint — that precedence was the field order of an object
|
||||
> literal and so could be neither named nor questioned — is what
|
||||
> `docs/REFACTORING.md` item 6 addresses. `Origin` now ranks
|
||||
> `default < env < session < prompt < param` as data, and every value carries the
|
||||
> origin it came from.
|
||||
|
||||
`README.md:107-113` stated:
|
||||
|
||||
@@ -185,6 +195,12 @@ Two pairs are inverted, and both have consequences:
|
||||
> schema key rather than only the defaulted ones; and `--use-defaults` refuses
|
||||
> to deploy a cube whose required key nothing supplied. Verified against all 22
|
||||
> cubes in `cubes/`: 19 build a complete `-D` run, the 3 below abort by name.
|
||||
>
|
||||
> Re-measured against the 25 cubes now in `packages/cubes-core/cubes`: 20 build a
|
||||
> complete `-D` run and 5 abort by name. Two of the additions are deliberate —
|
||||
> `user:add` lost its `PUBKEY` default (it was a specific personal key), and
|
||||
> `ssh:keygen` inherits that failure because it declares `dependencies: () =>
|
||||
> ['user:add']` and passes no parameters. See §6.6.
|
||||
|
||||
`README.md:99` states it outright; `README.md:105` claims defaults ensure "every
|
||||
cube has a predictable starting state".
|
||||
@@ -438,7 +454,11 @@ Public API in `src/index.ts` with no `API.md` entry: the entire history module
|
||||
`getHistoryPath`, `formatHistoryList`, `HISTORY_FILE`, `DEFAULT_HISTORY_SIZE`,
|
||||
plus `HistoryEntry` / `SessionHistory`), `BuildContext`,
|
||||
`runSessionReplayWorkflow`, `getConfigPaths`, `findCubeDirectories`, `getCube`,
|
||||
`filterInternalVariables`, `separateEnvAndCubeVariables`.
|
||||
~~`filterInternalVariables`, `separateEnvAndCubeVariables`~~ — those last two were
|
||||
dead on arrival (nothing called them once the session recorder stopped splitting
|
||||
`env` out) and have since been deleted rather than documented. `maskCommand`,
|
||||
`maskVariables`, `Variable`, `Variables`, `MASK` and the `Origin` / `Assignment`
|
||||
types are newly exported and also have no `API.md` entry.
|
||||
|
||||
The CLI cheat-sheet (`API.md:554-578`) omits `-R`, `-H`, `-P`, `--no-history`,
|
||||
and the `history` / `clear-history` commands.
|
||||
@@ -474,7 +494,20 @@ Because `loadCubes` turns each failure into an `errors` entry and `nopy.main.ts:
|
||||
aborts when `errors.length > 0`, a fresh clone cannot run a single cube. Neither
|
||||
README mentions a setup step.
|
||||
|
||||
### 4.2 🟠 The SSH password is printed in plaintext
|
||||
### 4.2 🟠 The SSH password is printed in plaintext — **mostly fixed**
|
||||
|
||||
> **Points 1 and 2 resolved; point 3 stands.** `maskCommand()`
|
||||
> (`nopy.executor.ts`) rewrites the SSH `--password` and every `--data` value the
|
||||
> manifest declared a secret, and it is applied at all three places the command
|
||||
> string is printed: the debug log, the dry-run plan, and `--print-only`. The
|
||||
> name heuristic described in point 2 is gone — the manifest's `secrets` array
|
||||
> says which keys are sensitive, so `TOKEN`, `PSK` and `AUTH_KEY` are covered
|
||||
> too, and it no longer matters that a key merely *looks* like a password.
|
||||
>
|
||||
> Point 3 is unchanged and now documented instead: the value still reaches
|
||||
> pyinfra on its command line, so it is visible in `ps`. That is inherent to
|
||||
> pyinfra's `--data` interface, not something nopy can mask. The shell-quoting
|
||||
> concern in the same point is also still open. See `docs/REFACTORING.md` item 7.
|
||||
|
||||
Not stated in any document, and it sits directly against the security notes at
|
||||
`README.md:217` and `:325` (which are narrowly about *storage*, and are correct
|
||||
@@ -500,7 +533,18 @@ That string is then:
|
||||
visible in the process list and, without quoting, vulnerable to shell
|
||||
metacharacters in the password.
|
||||
|
||||
### 4.3 🟠 History and session files record only prompted values
|
||||
### 4.3 ✅ History and session files record only prompted values — **fixed**
|
||||
|
||||
> **Resolved.** `buildDeployCall` records `Variables.persistable(cubeId)` — every
|
||||
> value the cube settled on, whatever its origin — so a `--use-defaults` run no
|
||||
> longer records an empty object, and a replay reproduces the run rather than
|
||||
> re-deriving it from whatever the defaults and `env` say at replay time. The one
|
||||
> deliberate exclusion is a key the manifest declared a secret; those are prompted
|
||||
> for again on replay, and a `-D` replay that would need one fails by name.
|
||||
>
|
||||
> The divergence noted at the end of this finding narrows but does not vanish: a
|
||||
> dependency graph that resolves differently can still produce a different
|
||||
> command, because `param` outranks `session` by design.
|
||||
|
||||
`README.md:414` says an entry records "the variable values that were answered at
|
||||
the prompts" — accurate, but the consequence is not drawn out.
|
||||
@@ -671,6 +715,29 @@ the three has to give.
|
||||
Covered under §1.5. `docs/API.md:160` documents the error; there is no code that
|
||||
raises it. Mutually dependent cubes recurse until the stack overflows.
|
||||
|
||||
### 6.6 🟠 `ssh:keygen` depends on `user:add` but shares nothing with it
|
||||
|
||||
`ssh:keygen` declares `dependencies: () => ['user:add']` with no parameters, so
|
||||
the two cubes resolve their `USER` independently:
|
||||
|
||||
- `ssh:keygen` defaults `USER` to `vagrant`;
|
||||
- `user:add` defaults `USER` to `` user${uniqid(5)} `` — a fresh random name.
|
||||
|
||||
So the dependency creates an account the dependent then ignores, and generates a
|
||||
key for a `vagrant` user it never created. Passing the value through
|
||||
(`[['user:add', {USER}]]`) is what the dependency spec exists for; `param`
|
||||
outranks `default`, so it would take effect.
|
||||
|
||||
Surfaced by removing `user:add`'s `PUBKEY` default: `ssh:keygen` now fails a `-D`
|
||||
run with `Cube "user:add" cannot run with --use-defaults: PUBKEY has no default
|
||||
value`, naming a cube the operator did not select. The underlying mismatch is
|
||||
older than that change and is not fixed here.
|
||||
|
||||
Related: `user:add`'s `USER` default is generated (`` user${uniqid(5)} ``), the
|
||||
same shape as the `PASSWORD` default that was removed. It is recorded in the
|
||||
session, so replays are stable, but each fresh `-D` run still creates a
|
||||
differently-named account.
|
||||
|
||||
---
|
||||
|
||||
## 7. Checked and accurate
|
||||
|
||||
+138
-23
@@ -21,19 +21,44 @@ shipped. If you only want to cut a release, jump to
|
||||
|
||||
## What ships
|
||||
|
||||
| Directory | Package | Binary |
|
||||
| ----------------- | ------------------ | -------- |
|
||||
| `packages/nopy` | `@bitsquare/nopy` | `nopy` |
|
||||
| `packages/keyman` | `@bitsquare/keyman` | `keyman` |
|
||||
| Directory | Package | Binary | Kind |
|
||||
| --------------------- | ----------------------- | -------- | ------------------------ |
|
||||
| `packages/nopy` | `@bitsquare/nopy` | `nopy` | CLI |
|
||||
| `packages/keyman` | `@bitsquare/keyman` | `keyman` | CLI |
|
||||
| `packages/nopy-cube` | `@bitsquare/nopy-cube` | — | library (cube authoring) |
|
||||
| `packages/cubes-core` | `@bitsquare/cubes-core` | — | cube bundle (no build) |
|
||||
|
||||
Both are ESM, both declare `engines.node >= 22`, and both expose a single
|
||||
All are ESM and declare `engines.node >= 22`. The two CLIs expose a single
|
||||
executable through `bin`, so `npm install -g` puts `nopy` / `keyman` on the
|
||||
`PATH`. `cubes/` is not a package and is never published.
|
||||
`PATH`; the other two are libraries you add to a project.
|
||||
|
||||
The tarball contents are pinned by `files: ["dist", "README.md", "LICENSE"]` —
|
||||
sources and tests are not shipped. `publishConfig.access: "public"` is what makes
|
||||
a scoped package publishable to npmjs without an extra flag; the workflows pass
|
||||
`--access public` anyway.
|
||||
The tarball contents are pinned by `files` — for the three TypeScript packages
|
||||
that is `["dist", "README.md", "LICENSE"]`, so sources and tests are not shipped.
|
||||
`cubes-core` ships `["cubes", "!cubes/**/*.log", "README.md", "LICENSE"]`: the
|
||||
negation matters, because a cube that has been run leaves a `pyinfra-debug.log`
|
||||
next to its `deploy.py`, and `.gitignore` does not filter an npm tarball.
|
||||
`publishConfig.access: "public"` is what makes a scoped package publishable to
|
||||
npmjs without an extra flag; the workflows pass `--access public` anyway.
|
||||
|
||||
### Dependencies between them
|
||||
|
||||
`keyman` stands alone. `nopy` and `cubes-core` both depend on `nopy-cube` through
|
||||
`workspace:*`, which drives three rules the rest of this document keeps coming
|
||||
back to:
|
||||
|
||||
1. **Publish with `pnpm`, not `npm`.** `link-workspace-packages` is unset and
|
||||
pnpm 10+ defaults it to `false`, so `workspace:*` is mandatory in the
|
||||
manifests. npm has no idea what that protocol is: `npm pack` copies the string
|
||||
through verbatim and the install fails with `EUNSUPPORTEDPROTOCOL`. `pnpm
|
||||
pack` and `pnpm publish` substitute the concrete version at pack time. Both
|
||||
workflows use `pnpm publish --ignore-scripts --no-git-checks`.
|
||||
2. **`nopy-cube` publishes before anything that depends on it.**
|
||||
`node scripts/publish-order.mjs` prints the publishable directories in
|
||||
dependency order — note that plain alphabetical `packages/*/` gets this
|
||||
backwards, putting `nopy` first.
|
||||
3. **Every packed manifest is checked.** `node scripts/verify-pack.mjs` packs
|
||||
each non-private package and fails if any `workspace:` range survived into the
|
||||
tarball. It runs in both publish workflows, after the build.
|
||||
|
||||
Versions and changelogs are maintained **by hand**. Nothing in CI commits a
|
||||
version bump, opens a release PR, or pushes a tag. A release happens because you
|
||||
@@ -47,7 +72,7 @@ All three live in [`.gitea/workflows`](.gitea/workflows) and run on the
|
||||
| Workflow | Trigger | Publishes |
|
||||
| ---------------------- | -------------------------------- | ------------------------------------------ |
|
||||
| `ci.yml` | pull requests, non-`main` pushes | nothing |
|
||||
| `publish-snapshot.yml` | pushes to `main` | **both** packages → Gitea, tag `main` |
|
||||
| `publish-snapshot.yml` | pushes to `main` | **every** package → Gitea, tag `main` |
|
||||
| `release.yml` | tags matching `*-v*` | **one** package → Gitea **and** npmjs |
|
||||
|
||||
`ci.yml` explicitly excludes `main` and all tags (`branches-ignore` +
|
||||
@@ -64,13 +89,17 @@ half-finished publish is worse than a slow queue.
|
||||
```
|
||||
checkout → pnpm → node → pnpm store cache → install
|
||||
→ lint:ci → typecheck → test:coverage → coverage summary
|
||||
→ build → npm pack --dry-run (per package) → upload coverage
|
||||
→ build → npm pack --dry-run (per package) → verify-pack
|
||||
→ upload coverage
|
||||
```
|
||||
|
||||
The `npm pack --dry-run --ignore-scripts` step prints the exact file list that
|
||||
would be published. It is there to catch a `files` or `bin` entry pointing at
|
||||
something the build no longer produces — a failure that would otherwise only
|
||||
surface after the version is already on a registry and immutable.
|
||||
surface after the version is already on a registry and immutable. `verify-pack`
|
||||
then packs for real and checks no `workspace:` range survived; both publish
|
||||
workflows run it too, but running it here is what puts the failure on the pull
|
||||
request rather than on the release.
|
||||
|
||||
Coverage HTML/JSON reports are uploaded as a `coverage` artifact with a 7-day
|
||||
retention. Both the artifact upload and the store cache are
|
||||
@@ -82,18 +111,25 @@ slower CI rather than broken CI.
|
||||
```
|
||||
checkout → pnpm → node → cache → install
|
||||
→ lint:ci → typecheck → test:coverage → coverage summary → build
|
||||
→ write .npmrc → publish both packages → delete .npmrc
|
||||
→ verify-pack → write .npmrc → publish every package → delete .npmrc
|
||||
```
|
||||
|
||||
One job, no `needs:` barrier, so install and build happen exactly once and
|
||||
nothing has to be passed between jobs as an artifact.
|
||||
|
||||
The publish step is **two passes** over `node scripts/publish-order.mjs`: the
|
||||
first stamps the snapshot version into every manifest with `npm pkg set`, the
|
||||
second publishes. They cannot be one loop — `pnpm publish` reads a linked
|
||||
package's version out of its manifest at pack time, so stamping and publishing
|
||||
one package at a time would bake the *old* `nopy-cube` version into `nopy`'s
|
||||
tarball.
|
||||
|
||||
### `release.yml`
|
||||
|
||||
```
|
||||
checkout → resolve tag → check secrets
|
||||
→ pnpm → node → cache → install
|
||||
→ lint:ci → typecheck → test:coverage → build
|
||||
→ pnpm → node → cache → install → check linked deps are released
|
||||
→ lint:ci → typecheck → test:coverage → build → verify-pack
|
||||
→ publish to Gitea → publish to npmjs → delete .npmrc
|
||||
→ create the Gitea release → step summary
|
||||
```
|
||||
@@ -102,6 +138,14 @@ Tag resolution and the secret check run **before** anything is installed or
|
||||
built, so a malformed tag or a missing token fails in seconds instead of after
|
||||
the whole gate.
|
||||
|
||||
*Check linked deps are released* asks npmjs whether every `workspace:` dependency
|
||||
of the package being released already exists at the version pnpm is about to
|
||||
bake in (`scripts/linked-deps.mjs` → `npm view`). Tagging `nopy-v1.3.0` while
|
||||
`@bitsquare/nopy-cube@1.1.0` is still unpublished would otherwise ship a tarball
|
||||
nobody can install, and npmjs only lets you unpublish for 72 hours. The check is
|
||||
npmjs-only: it runs before any credentials are written, and npmjs is the registry
|
||||
where the mistake is permanent.
|
||||
|
||||
## The verification gate
|
||||
|
||||
The same three commands guard every path into a registry:
|
||||
@@ -143,7 +187,7 @@ a prerelease over `latest` by accident.
|
||||
|
||||
## Snapshots
|
||||
|
||||
Every commit that lands on `main` publishes both packages to the Gitea registry,
|
||||
Every commit that lands on `main` publishes every package to the Gitea registry,
|
||||
versioned as:
|
||||
|
||||
```
|
||||
@@ -175,7 +219,34 @@ edit is discarded with the workspace and is never committed.
|
||||
```
|
||||
|
||||
The tag name is `<directory>-v<version>` — the directory under `packages/`, not
|
||||
the npm name. `nopy-v1.2.0`, not `@bitsquare/nopy-v1.2.0`.
|
||||
the npm name. `nopy-v1.2.0`, not `@bitsquare/nopy-v1.2.0`. All four prefixes work
|
||||
the same way:
|
||||
|
||||
```sh
|
||||
git tag nopy-v1.2.0
|
||||
git tag keyman-v1.2.0
|
||||
git tag nopy-cube-v1.2.0
|
||||
git tag cubes-core-v1.2.0
|
||||
```
|
||||
|
||||
### Ordering when more than one package changed
|
||||
|
||||
Tags are independent, but the dependency graph is not. If a release touches
|
||||
`nopy-cube` *and* something that depends on it, release them in this order,
|
||||
waiting for each run to go green:
|
||||
|
||||
```
|
||||
nopy-cube → nopy, cubes-core (these two are independent of each other)
|
||||
```
|
||||
|
||||
Release `nopy` first and the run stops at the *check linked deps* step, telling
|
||||
you the `nopy-cube` version it wanted is not on npmjs. That is the guard working;
|
||||
release `nopy-cube`, then re-tag. `node scripts/publish-order.mjs` prints the
|
||||
order if you would rather not reason about it.
|
||||
|
||||
Bumping `nopy-cube` means bumping the packages that depend on it in the same
|
||||
change — the `workspace:*` range resolves to whatever version is in the workspace
|
||||
at pack time, so their next release picks it up whether or not you meant it to.
|
||||
|
||||
The tag decides **which** package ships; `package.json` decides the **version**.
|
||||
The workflow re-reads the manifest and refuses to continue if the two disagree:
|
||||
@@ -281,12 +352,21 @@ file written into the workspace can never be committed by accident.
|
||||
|
||||
## Installing the packages
|
||||
|
||||
From npmjs — public, no configuration:
|
||||
From npmjs — public, no configuration. The CLIs go on the `PATH`:
|
||||
|
||||
```sh
|
||||
npm install -g @bitsquare/nopy @bitsquare/keyman
|
||||
```
|
||||
|
||||
The other two go into a project. A cube bundle is a dev dependency of whatever
|
||||
repo describes your infrastructure; `nopy-cube` is only needed if you are writing
|
||||
cubes of your own:
|
||||
|
||||
```sh
|
||||
pnpm add -D @bitsquare/cubes-core # then name it in .nopyrc.json cubePackages
|
||||
pnpm add -D @bitsquare/nopy-cube zod # authoring your own manifests
|
||||
```
|
||||
|
||||
From the Gitea registry, which holds every snapshot plus a mirror of every
|
||||
release. Per-project, in the repo's `.npmrc`:
|
||||
|
||||
@@ -321,9 +401,17 @@ refuses to overwrite an existing version — without the check, a re-run would f
|
||||
on the first registry and never reach the second.
|
||||
|
||||
**The build is explicit, publishes are `--ignore-scripts`.** `prepack` exists for
|
||||
humans running `npm pack` locally; in CI the build has already run as its own
|
||||
step, and repeating it inside `npm publish` would only cost time and add a way
|
||||
for a lifecycle script to change what ships after the gate looked at it.
|
||||
humans packing locally; in CI the build has already run as its own step, and
|
||||
repeating it inside `pnpm publish` would only cost time and add a way for a
|
||||
lifecycle script to change what ships after the gate looked at it.
|
||||
|
||||
**`verify-pack.mjs` checks the artefact, not the source.** Reading `package.json`
|
||||
in the repo would only tell you what you already know — every one of them says
|
||||
`workspace:*`. The question is what pnpm wrote into the tarball, so the script
|
||||
packs, extracts `package/package.json`, and reads the ranges back out. It cannot
|
||||
pass `--ignore-scripts`, because `pnpm pack` has no such flag (only `pnpm publish`
|
||||
does), so `prepack` rebuilds — which at least means the tarball under test is
|
||||
byte-for-byte the one publish would ship.
|
||||
|
||||
**One job per workflow.** No artifact hand-off, no second install, no risk of
|
||||
publishing a tree that a different job built.
|
||||
@@ -352,7 +440,31 @@ See what would actually be in the tarball:
|
||||
|
||||
```sh
|
||||
pnpm run build
|
||||
cd packages/nopy && npm pack --dry-run --ignore-scripts
|
||||
cd packages/nopy && pnpm pack --dry-run
|
||||
```
|
||||
|
||||
Check that no `workspace:` range leaks into a published manifest — the same
|
||||
check CI runs:
|
||||
|
||||
```sh
|
||||
node scripts/verify-pack.mjs
|
||||
node scripts/publish-order.mjs # the order to release in
|
||||
node scripts/linked-deps.mjs packages/nopy # what must be on the registry first
|
||||
```
|
||||
|
||||
Rehearse an install the way a stranger gets one, without publishing anything.
|
||||
Use **npm**, not pnpm: npm is the one that rejects a leaked `workspace:` range,
|
||||
so a clean install here is the real proof.
|
||||
|
||||
```sh
|
||||
pnpm --filter @bitsquare/nopy-cube pack --pack-destination /tmp/tgz
|
||||
pnpm --filter @bitsquare/nopy pack --pack-destination /tmp/tgz
|
||||
pnpm --filter @bitsquare/cubes-core pack --pack-destination /tmp/tgz
|
||||
|
||||
mkdir /tmp/try && cd /tmp/try && npm init -y
|
||||
npm install /tmp/tgz/*.tgz
|
||||
echo '{"hosts":["h"],"cubePackages":["@bitsquare/cubes-core"]}' > .nopyrc.json
|
||||
./node_modules/.bin/nopy install -l session.json -P -D
|
||||
```
|
||||
|
||||
Try the binary as an end user would get it, without publishing:
|
||||
@@ -384,6 +496,9 @@ npm view @bitsquare/nopy@1.2.0 version \
|
||||
| `npm pack --dry-run` step fails | A `files` or `bin` path no longer exists after the build. Fix before it reaches a registry. |
|
||||
| Snapshot workflow green, nothing installable | Snapshots are only on Gitea and only under `@main`. Point the scope at the Gitea registry. |
|
||||
| The release workflow did not trigger | The tag must match `*-v*` and must be pushed (`git push origin <tag>`), not just created. |
|
||||
| `EUNSUPPORTEDPROTOCOL` / `Unsupported URL Type "workspace:"` on install | A `workspace:` range reached a tarball — something published with `npm publish` instead of `pnpm publish`. `verify-pack.mjs` exists to catch this before it ships. |
|
||||
| `... is not published yet on npmjs` before the gate runs | Releasing a package before its `nopy-cube` dependency. Tag and release `nopy-cube` first, then re-tag. |
|
||||
| `verify-pack.mjs` fails locally with a build error | `pnpm pack` runs `prepack`, so a broken build fails the check. Fix the build; there is no skip flag. |
|
||||
|
||||
## Recovering from a bad publish
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
id: 'user:add',
|
||||
name: 'Add a user with fish shell and tools',
|
||||
dependencies: () => ['apt:essentials'],
|
||||
schema: z.object({
|
||||
USER: z
|
||||
.string()
|
||||
.describe('Username for the new user account')
|
||||
.default(() => `user${cubes.uniqid(5)}`),
|
||||
PASSWORD: z.string().describe('Password for the new user account').default(cubes.uniqid),
|
||||
GROUPS: z
|
||||
.string()
|
||||
.describe('Comma-separated list of additional groups (e.g., "docker,sudo")')
|
||||
.default(''),
|
||||
PUBKEY: z
|
||||
.string()
|
||||
.describe('SSH public key to authorize for the user')
|
||||
.default(
|
||||
'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICpnZ6IxwQKL1rBE4dy7w5Sd3s2tLFZUDfjH87C1QIlc bdiedrichsen@Benjamins-MBP.lan'
|
||||
),
|
||||
}),
|
||||
});
|
||||
+2
-1
@@ -13,7 +13,7 @@
|
||||
"test": "pnpm -r run test",
|
||||
"test:coverage": "pnpm -r run test:coverage",
|
||||
"coverage:summary": "node scripts/coverage-summary.mjs",
|
||||
"typecheck": "tsc --build --noEmit",
|
||||
"typecheck": "tsc --build",
|
||||
"lint": "biome check .",
|
||||
"lint:fix": "biome check --write .",
|
||||
"lint:ci": "biome ci .",
|
||||
@@ -26,6 +26,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.5.5",
|
||||
"@bitsquare/cubes-core": "workspace:*",
|
||||
"@logtape/logtape": "^2.2.4",
|
||||
"@types/node": "^26.1.1",
|
||||
"simple-git-hooks": "^2.13.1",
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 bitsquare
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,59 @@
|
||||
# @bitsquare/cubes-core
|
||||
|
||||
The core cube bundle for [nopy](https://www.npmjs.com/package/@bitsquare/nopy):
|
||||
base packages, users, SSH, firewalling, networking, web serving and runtimes.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
pnpm add -D @bitsquare/cubes-core
|
||||
```
|
||||
|
||||
Then name it in `.nopyrc.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"hosts": ["web-1"],
|
||||
"cubePackages": ["@bitsquare/cubes-core"]
|
||||
}
|
||||
```
|
||||
|
||||
`nopy` resolves the package from the directory of the config file that named it,
|
||||
reads `nopy.cubes` out of its `package.json`, and scans those directories exactly
|
||||
as it scans a `cubeDirs` entry. Nothing has to be linked or copied.
|
||||
|
||||
## What is in it
|
||||
|
||||
| Area | Cube ids |
|
||||
| ---------- | ------------------------------------------------------------------- |
|
||||
| admin | `admin:cockpit`, `admin:hostname`, `admin:locale` |
|
||||
| packages | `apt:essentials`, `apt:install` |
|
||||
| hardening | `armor:fail2ban`, `armor:ssh`, `armor:ufw` |
|
||||
| web | `caddy`, `caddy:spa` |
|
||||
| source | `git:clone` |
|
||||
| networking | `net:tailscale`, `net:wifi:access-point`, `net:wifi:connection` |
|
||||
| runtimes | `runtime:docker`, `runtime:nodevm` |
|
||||
| services | `service:autostart` |
|
||||
| ssh | `ssh:authorize`, `ssh:keygen`, `ssh:keyman` |
|
||||
| users | `user:add`, `user:edit` |
|
||||
|
||||
Run `nopy` and pick from the list, or `nopy -P` to print the pyinfra commands
|
||||
without executing them. Each cube directory has its own `README.md`.
|
||||
|
||||
## Cube ids are global
|
||||
|
||||
An id such as `apt:essentials` is claimed repo-wide, not per bundle: two cubes
|
||||
with the same id — whichever sources they came from — abort the run with an
|
||||
error naming both. Prefix your own cubes distinctly if you also point
|
||||
`cubeDirs` at a local tree.
|
||||
|
||||
## The bundle is read-only
|
||||
|
||||
Under pnpm the installed files are hardlinked into the global store, so a cube
|
||||
that writes next to its own `deploy.py` corrupts that store for every project on
|
||||
the machine. Cubes here write to `/tmp` or to the remote host, never to their
|
||||
own directory.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'admin:cockpit',
|
||||
name: 'Install cockpit and utils',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
import { cubes, uniqid } from '@bitsquare/nopy';
|
||||
import { Manifest, uniqid } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Manifest for the admin:hostname cube.
|
||||
* This cube allows for setting and persistently changing the system's hostname.
|
||||
*/
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'admin:hostname',
|
||||
name: 'Permanently change the hostname',
|
||||
dependencies: () => [],
|
||||
@@ -12,9 +12,9 @@ Configures system keyboard layout permanently by updating `/etc/default/keyboard
|
||||
## Usage
|
||||
|
||||
```javascript
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
name: 'My Host Setup',
|
||||
dependencies: () => [
|
||||
['admin:locale', { LAYOUT: 'de' }]
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'admin:locale',
|
||||
name: 'Configure system locale and keyboard layout',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'apt:essentials',
|
||||
name: 'Install essential packages',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'apt:install',
|
||||
name: 'Install packages with apt',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'armor:fail2ban',
|
||||
name: 'Install and enable fail2ban',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'armor:ssh',
|
||||
name: 'Secure SSH server by disabling password authentication',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'armor:ufw',
|
||||
name: 'Activate ufw (uncomplicated firewall)',
|
||||
dependencies: () => ['apt:essentials'],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'caddy',
|
||||
name: 'Install Caddy webserver',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'caddy:spa',
|
||||
name: 'Install single page application',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'git:clone',
|
||||
name: 'Clone a repository',
|
||||
dependencies: () => [],
|
||||
+7
-1
@@ -13,7 +13,7 @@ Installs and authenticates the Tailscale client on a Linux host.
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `AUTH_KEY` | `""` | Tailscale Auth Key (recommended to use a 'reusable' or 'ephemeral' key). |
|
||||
| `AUTH_KEY` | `""` | **Secret.** Tailscale Auth Key (recommended to use a 'reusable' or 'ephemeral' key). |
|
||||
| `LOGIN_SERVER` | `https://controlplane.tailscale.com` | The coordination server URL. Set this to your Headscale instance URL if applicable. |
|
||||
| `EXTRA_ARGS` | `""` | Additional flags to pass to `tailscale up` (e.g., `--advertise-exit-node`). |
|
||||
| `FORCE_REAUTH` | `false` | If true, forces the client to re-authenticate. |
|
||||
@@ -25,3 +25,9 @@ nopy install tailscale
|
||||
```
|
||||
|
||||
When prompted, provide your `AUTH_KEY`. If you are using Headscale, also provide the `LOGIN_SERVER` URL.
|
||||
|
||||
`AUTH_KEY` is declared in the manifest's `secrets`, so nopy keeps it out of session
|
||||
and history files and masks it in any command it prints. It is asked for again on
|
||||
replay, and a `--use-defaults` replay refuses rather than joining the tailnet with
|
||||
an empty key. Prefer an ephemeral key regardless — the value is still on pyinfra's
|
||||
command line while the deployment runs.
|
||||
+3
-2
@@ -1,10 +1,11 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'net:tailscale',
|
||||
name: 'Install and authenticate Tailscale',
|
||||
dependencies: () => ['apt:essentials'],
|
||||
secrets: ['AUTH_KEY'],
|
||||
schema: z.object({
|
||||
AUTH_KEY: z.string().describe('Tailscale Auth Key for headless authentication').default(''),
|
||||
LOGIN_SERVER: z
|
||||
+12
-3
@@ -19,10 +19,19 @@ Configures a Linux device as a WiFi Access Point using NetworkManager's `nmcli`
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
### Required
|
||||
> **This section is out of date** — it lists parameters the manifest does not
|
||||
> declare (`NETWORK_DEVICE`, `CHANNEL`, `IP_ADDRESS`) and omits `AP_IP`. Read
|
||||
> `manifest.mjs` for the real list. Tracked as §5 in the repository's
|
||||
> `DOCS-AUDIT.md`.
|
||||
|
||||
- **SSID**: WiFi network name (1-32 characters)
|
||||
- **PASSWORD**: WPA2 password (8-63 characters)
|
||||
### Prompted first
|
||||
|
||||
- **SSID**: WiFi network name (1-32 characters). Defaults to `PiPoint`.
|
||||
- **PASSWORD**: WPA2 password (8-63 characters). Defaults to `1223334444` — a
|
||||
placeholder that should not survive contact with a real network.
|
||||
Declared in the manifest's `secrets`, so nopy keeps it out of session and
|
||||
history files and masks it in printed commands, and re-prompts on replay. The
|
||||
value is still on pyinfra's command line, so it is visible in `ps` during the run.
|
||||
|
||||
### Optional (with defaults)
|
||||
|
||||
+3
-2
@@ -1,10 +1,11 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'net:wifi:access-point',
|
||||
name: 'Configure WiFi Access Point (NetworkManager)',
|
||||
dependencies: () => [],
|
||||
secrets: ['PASSWORD'],
|
||||
schema: z.object({
|
||||
SSID: z.string().min(1).max(32).default('PiPoint').describe('WiFi network name (SSID)'),
|
||||
PASSWORD: z
|
||||
+5
@@ -42,6 +42,11 @@ nopy install network:wifi:connection --env SSID="OfficeWiFi" --env PASSWORD="pas
|
||||
|
||||
## Security Notes
|
||||
|
||||
- `PASSWORD` is declared in the manifest's `secrets`: nopy keeps it out of session
|
||||
and history files and masks it in every command it prints. It is prompted for
|
||||
again on replay.
|
||||
- That covers what nopy writes, not everything. The value is still on pyinfra's
|
||||
command line, so it is visible in `ps` while the deployment runs.
|
||||
- WiFi passwords will be stored in `/etc/NetworkManager/system-connections/` on the target host.
|
||||
- Passing passwords via `--env` may leave them in your local shell history.
|
||||
|
||||
+3
-2
@@ -1,4 +1,4 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
// [agnt://cogen/cogen/network-wifi-connection-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"}
|
||||
@@ -7,9 +7,10 @@ import { z } from 'zod';
|
||||
* Manifest for the network:wifi:connection cube.
|
||||
* Configures a WiFi client connection using NetworkManager (nmcli).
|
||||
*/
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'net:wifi:connection',
|
||||
name: 'network:wifi:connection - Connect to a WiFi network',
|
||||
secrets: ['PASSWORD'],
|
||||
schema: z.object({
|
||||
SSID: z.string().min(1).describe('The SSID of the WiFi network to connect to'),
|
||||
PASSWORD: z.string().min(8).describe('The password for the WiFi network'),
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'runtime:docker',
|
||||
name: 'Install docker and tools',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'runtime:nodevm',
|
||||
name: 'Install nvm and nodejs with global packages',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'service:autostart',
|
||||
name: 'Manage systemd service autostart',
|
||||
dependencies: () => [],
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'ssh:authorize',
|
||||
name: 'Authorize SSH public key for a user',
|
||||
dependencies: () => [],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'ssh:keygen',
|
||||
name: 'Generate SSH key for a given $USER',
|
||||
dependencies: () => ['user:add'],
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'ssh:keyman',
|
||||
name: 'Deploy an ssh key managed by keyman',
|
||||
dependencies: () => [],
|
||||
@@ -36,9 +36,15 @@ This cube creates a new user account with a modern shell environment (Fish), SSH
|
||||
- Username for the new user account
|
||||
- Default: `userXXXXX` (randomly generated 5-character suffix)
|
||||
|
||||
- **PASSWORD** (string, auto-generated)
|
||||
- **PASSWORD** (string, **secret**)
|
||||
- Password for the new user account
|
||||
- Default: randomly generated secure password
|
||||
- Default: the literal `changeme` — a placeholder, not a credential. Change it
|
||||
on first login, or pass a real one.
|
||||
- Declared in the manifest's `secrets`, so it is never written to a session or
|
||||
history file and is masked in printed commands. A replay asks for it again.
|
||||
- It used to default to a randomly generated password. That was removed: since
|
||||
the value is not recorded, an unattended run created an account with a
|
||||
credential nobody had seen, and replaying that run produced a different one.
|
||||
|
||||
- **GROUPS** (string, default: `''`)
|
||||
- Comma-separated list of additional groups (e.g., `"docker,sudo"`)
|
||||
@@ -47,9 +53,16 @@ This cube creates a new user account with a modern shell environment (Fish), SSH
|
||||
- `sudo` - Administrative privileges
|
||||
- `www-data` - Web server file access
|
||||
|
||||
- **PUBKEY** (string, has default)
|
||||
- **PUBKEY** (string, **required** — no default)
|
||||
- SSH public key to authorize for the user
|
||||
- Should be your public key for passwordless SSH access
|
||||
- There is deliberately no default. It used to be a specific personal key, so
|
||||
accepting the default authorized *someone else's* key on the new account.
|
||||
No key would be a sensible guess, so the cube asks instead.
|
||||
- Because it is required, `--use-defaults` refuses to run this cube unless
|
||||
`PUBKEY` comes from `env` in `.nopyrc.json`, a dependency, or a hook.
|
||||
- Submitting an empty value at the prompt authorizes no key at all (the account
|
||||
is still created, with password login only).
|
||||
|
||||
## Dependencies
|
||||
|
||||
@@ -7,7 +7,10 @@ USER = host.data.USER
|
||||
HOME_DIR = f"/home/{USER}"
|
||||
TMP_DIR = f"{HOME_DIR}/tmp"
|
||||
PASSWORD = host.data.PASSWORD
|
||||
# An empty submission at the prompt must not become an empty authorized_keys
|
||||
# line, so an absent key means no key rather than a blank one.
|
||||
PUBKEY = host.data.PUBKEY
|
||||
PUBKEYS = [PUBKEY] if PUBKEY and str(PUBKEY).strip() else []
|
||||
GROUPS = list(filter(None, map(str.strip, str(host.data.GROUPS).split())))
|
||||
FISH_PATH = "/usr/bin/fish"
|
||||
FISH_CONFIG_DIR = f"{HOME_DIR}/.config/fish"
|
||||
@@ -31,7 +34,7 @@ server.user(
|
||||
create_home=True,
|
||||
groups=GROUPS,
|
||||
shell=FISH_PATH,
|
||||
public_keys=[PUBKEY],
|
||||
public_keys=PUBKEYS,
|
||||
_sudo=True
|
||||
)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Manifest, uniqid } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default Manifest({
|
||||
id: 'user:add',
|
||||
name: 'Add a user with fish shell and tools',
|
||||
dependencies: () => ['apt:essentials'],
|
||||
secrets: ['PASSWORD'],
|
||||
schema: z.object({
|
||||
USER: z
|
||||
.string()
|
||||
.describe('Username for the new user account')
|
||||
.default(() => `user${uniqid(5)}`),
|
||||
// A fixed placeholder, not a generated one: the password is never recorded
|
||||
// in a session, so a generated default meant every run produced credentials
|
||||
// nobody had seen and a replay produced different ones again.
|
||||
PASSWORD: z.string().describe('Password for the new user account').default('changeme'),
|
||||
GROUPS: z
|
||||
.string()
|
||||
.describe('Comma-separated list of additional groups (e.g., "docker,sudo")')
|
||||
.default(''),
|
||||
// No default on purpose. This used to carry a specific personal key, which
|
||||
// meant an unattended run authorised someone else's key on the new account.
|
||||
// Leaving it required makes `--use-defaults` refuse by name instead of
|
||||
// guessing, and there is no key that would be a sensible guess.
|
||||
PUBKEY: z.string().describe('SSH public key to authorize for the user'),
|
||||
}),
|
||||
});
|
||||
@@ -18,7 +18,7 @@ This cube allows you to update existing user accounts on the target system. It c
|
||||
| Variable | Type | Description | Required |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `USER` | `string` | The username of the account to modify | Yes |
|
||||
| `PASSWORD` | `string` | New password for the user | No |
|
||||
| `PASSWORD` | `string` | New password for the user. **Secret**: never recorded in a session or history file, masked in printed commands, re-prompted on replay. | No |
|
||||
| `GROUPS` | `string` | Comma-separated list of groups to ADD (e.g., `docker,sudo`) | No |
|
||||
| `GROUPS_ABSENT` | `string` | Comma-separated list of groups to REMOVE | No |
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { cubes } from '@bitsquare/nopy';
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
// [agnt://cogen/cogen/user-edit-1]{cartridge: "ansiblings/cubes", action: "generated", status: "generated"}
|
||||
@@ -7,10 +7,11 @@ import { z } from 'zod';
|
||||
* Manifest for the user:edit cube.
|
||||
* Allows modifying existing user accounts (password, groups).
|
||||
*/
|
||||
export default cubes.Manifest({
|
||||
export default Manifest({
|
||||
id: 'user:edit',
|
||||
name: 'user:edit - Modify an existing user account',
|
||||
dependencies: () => [],
|
||||
secrets: ['PASSWORD'],
|
||||
schema: z.object({
|
||||
USER: z.string().describe('The username of the account to modify'),
|
||||
PASSWORD: z.string().optional().describe('New password for the user (optional)'),
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"name": "@bitsquare/cubes-core",
|
||||
"version": "1.0.0-alpha0",
|
||||
"description": "The core nopy cube bundle: apt, users, ssh, networking, services and runtimes.",
|
||||
"keywords": [
|
||||
"nopy",
|
||||
"nopy-cubes",
|
||||
"pyinfra",
|
||||
"deployment",
|
||||
"infrastructure"
|
||||
],
|
||||
"license": "MIT",
|
||||
"author": "bitsquare",
|
||||
"type": "module",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://gitea.bitsquare.dev/BitSquare/ansiblings.git",
|
||||
"directory": "packages/cubes-core"
|
||||
},
|
||||
"homepage": "https://gitea.bitsquare.dev/BitSquare/ansiblings/src/branch/main/packages/cubes-core",
|
||||
"bugs": {
|
||||
"url": "https://gitea.bitsquare.dev/BitSquare/ansiblings/issues"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"nopy": {
|
||||
"cubes": [
|
||||
"./cubes"
|
||||
]
|
||||
},
|
||||
"files": [
|
||||
"cubes",
|
||||
"!cubes/**/*.log",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bitsquare/nopy-cube": "workspace:*",
|
||||
"zod": "^4.4.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 bitsquare
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,69 @@
|
||||
# @bitsquare/nopy-cube
|
||||
|
||||
The authoring surface for [nopy](https://www.npmjs.com/package/@bitsquare/nopy)
|
||||
cubes — the `Manifest` factory, the `Cube` class, and the types around them.
|
||||
|
||||
A cube manifest ships nothing but data, so it should not have to depend on a CLI
|
||||
to describe itself. This package is what a **cube bundle** depends on: no
|
||||
`commander`, no `inquirer`, no `execa`, no process spawning. `@bitsquare/nopy`
|
||||
re-exports everything here, so a manifest that already imports from
|
||||
`@bitsquare/nopy` keeps working unchanged.
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
pnpm add @bitsquare/nopy-cube zod
|
||||
```
|
||||
|
||||
`zod` is a **peer dependency** on purpose: the manifest, the schema it builds and
|
||||
the `Manifest` factory should all see the same copy.
|
||||
|
||||
## Writing a manifest
|
||||
|
||||
```js
|
||||
// cubes/net/tailscale/manifest.mjs
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default Manifest({
|
||||
id: 'net:tailscale',
|
||||
name: 'Tailscale',
|
||||
schema: z.object({
|
||||
AUTH_KEY: z.string().describe('Tailscale auth key'),
|
||||
ACCEPT_ROUTES: z.boolean().describe('Accept advertised routes').default(true),
|
||||
}),
|
||||
secrets: ['AUTH_KEY'],
|
||||
dependencies: (vars) => (vars.ACCEPT_ROUTES ? ['net:ip-forwarding'] : []),
|
||||
before: [async (ctx, vars) => ctx.exec('apt:essentials', {})],
|
||||
});
|
||||
```
|
||||
|
||||
Every schema field should carry a `.describe()` — nopy uses it as the prompt
|
||||
label — and a `.default()` wherever a sensible one exists, so `--use-defaults`
|
||||
can run the cube without prompting.
|
||||
|
||||
`secrets` names the schema keys that hold sensitive values. Nopy keeps those out
|
||||
of session and history files and masks them in every command it prints; it does
|
||||
not infer them, so a key nothing declares is recorded and printed in the clear.
|
||||
Each entry must be a key of `schema` — naming anything else is a manifest error.
|
||||
Give a secret a placeholder `.default()` rather than a real credential: a default
|
||||
lives in the manifest, where none of that protection reaches it.
|
||||
|
||||
The manifest lives next to a `deploy.py` in the same directory; together they
|
||||
make a cube. See the
|
||||
[nopy README](https://www.npmjs.com/package/@bitsquare/nopy) for the full cube
|
||||
contract and for how to publish a directory of cubes as a bundle.
|
||||
|
||||
## Exports
|
||||
|
||||
| Export | What it is |
|
||||
| ----------------------------------- | -------------------------------------------------------------- |
|
||||
| `Manifest(opts)` | Builds a manifest, filling in `id`, `schema`, `secrets`, `before`, `after` |
|
||||
| `createManifest` / `manifest` | Aliases of `Manifest` |
|
||||
| `Cube` | A loaded manifest plus its directory; `getDefaults()`, `requiredKeys()`, `secrets`, `isSecret()` |
|
||||
| `zodKind` / `zodInner` | Instance-agnostic zod introspection, safe across zod copies |
|
||||
| `AnyObjectSchema`, `CubeVariables`, `DependencySpec`, `Hook`, `HookContext`, `CubeSource`, `LoadResult` | types |
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"name": "@bitsquare/nopy-cube",
|
||||
"version": "1.0.0-alpha0",
|
||||
"description": "Authoring types for nopy cubes: the Manifest factory and the Cube contract.",
|
||||
"keywords": [
|
||||
"nopy",
|
||||
"pyinfra",
|
||||
"deployment",
|
||||
"infrastructure"
|
||||
],
|
||||
"license": "MIT",
|
||||
"author": "bitsquare",
|
||||
"type": "module",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://gitea.bitsquare.dev/BitSquare/ansiblings.git",
|
||||
"directory": "packages/nopy-cube"
|
||||
},
|
||||
"homepage": "https://gitea.bitsquare.dev/BitSquare/ansiblings/src/branch/main/packages/nopy-cube",
|
||||
"bugs": {
|
||||
"url": "https://gitea.bitsquare.dev/BitSquare/ansiblings/issues"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rm -rf dist .tsbuildinfo",
|
||||
"build": "tsc",
|
||||
"prepack": "pnpm run build",
|
||||
"link:local": "pnpm run build && npm link",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^26.1.1",
|
||||
"@vitest/coverage-v8": "^4.1.10",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^4.1.10",
|
||||
"zod": "^4.4.3"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Factory functions for creating cube configurations
|
||||
* @module cubes/factories
|
||||
* @module factories
|
||||
*/
|
||||
|
||||
import { type AnyObjectSchema, Manifest } from './types.js';
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* @bitsquare/nopy-cube — the authoring surface for nopy cubes.
|
||||
*
|
||||
* Everything a `manifest.mjs` needs and nothing else: no CLI, no prompts, no
|
||||
* process spawning. `@bitsquare/nopy` re-exports all of it, so a manifest can
|
||||
* import from either package.
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
export {
|
||||
createManifest,
|
||||
ManifestFactory,
|
||||
manifest,
|
||||
} from './factories.js';
|
||||
export type {
|
||||
AnyObjectSchema,
|
||||
CubeSource,
|
||||
CubeVariables,
|
||||
DependencySpec,
|
||||
Hook,
|
||||
HookContext,
|
||||
LoadResult,
|
||||
} from './types.js';
|
||||
export {
|
||||
Cube,
|
||||
Manifest,
|
||||
zodInner,
|
||||
zodKind,
|
||||
} from './types.js';
|
||||
export { uniqid } from './utils.js';
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Type definitions for Nopy cubes
|
||||
* @module cubes/types
|
||||
* @module types
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
@@ -47,6 +47,18 @@ export interface Manifest<Schema extends AnyObjectSchema = AnyObjectSchema> {
|
||||
name: string;
|
||||
/** Zod schema for validating cube variables */
|
||||
schema: Schema;
|
||||
/**
|
||||
* Schema keys holding secrets. Their values are never written to a session
|
||||
* file, and are masked wherever a command or a variable would be printed.
|
||||
*
|
||||
* A plain array rather than schema-level metadata on purpose: `.meta()` and
|
||||
* `.describe()` both store into zod's global registry, which is per-copy — a
|
||||
* manifest that builds its schema with its own zod writes the marker into a
|
||||
* registry this process cannot read. A missed `.describe()` costs an ugly
|
||||
* prompt label; a missed secret marker writes a password to disk, so this one
|
||||
* cannot be allowed to fail open. See {@link zodKind} for the same hazard.
|
||||
*/
|
||||
secrets?: string[];
|
||||
/** Dynamic dependency resolver based on collected variables */
|
||||
dependencies?: (variables: z.infer<Schema>) => DependencySpec[];
|
||||
/** Hooks to run before cube execution */
|
||||
@@ -65,6 +77,7 @@ export function Manifest<Schema extends AnyObjectSchema>(
|
||||
id: opts.id ?? '',
|
||||
name: opts.name,
|
||||
schema: opts.schema ?? (z.object({}) as unknown as Schema),
|
||||
secrets: opts.secrets ?? [],
|
||||
dependencies: opts.dependencies,
|
||||
before: opts.before ?? [],
|
||||
after: opts.after ?? [],
|
||||
@@ -189,6 +202,15 @@ export class Cube<Schema extends AnyObjectSchema = AnyObjectSchema> {
|
||||
.filter(([, zodType]) => !zodType.safeParse(undefined).success)
|
||||
.map(([key]) => key);
|
||||
}
|
||||
|
||||
/** Schema keys the manifest declared as secrets. */
|
||||
get secrets(): string[] {
|
||||
return this.manifest.secrets ?? [];
|
||||
}
|
||||
|
||||
isSecret(key: string): boolean {
|
||||
return this.secrets.includes(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Utility functions for cubes
|
||||
* @module cubes/utils
|
||||
* @module utils
|
||||
*/
|
||||
|
||||
/**
|
||||
+2
-2
@@ -1,10 +1,10 @@
|
||||
/**
|
||||
* Tests for cubes/factories module
|
||||
* Tests for the manifest factories
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
import { createManifest, manifest } from '../src/cubes/factories.js';
|
||||
import { createManifest, manifest } from '../src/factories.js';
|
||||
|
||||
describe('createManifest', () => {
|
||||
it('creates manifest with basic properties', () => {
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* A schema that behaves like zod's but does not share zod's prototypes.
|
||||
*
|
||||
* Once cubes arrive from `node_modules`, the schema a manifest builds may come
|
||||
* from a *second* copy of zod — its own dependency, or one shipped inside a
|
||||
* bundle. Such a schema is structurally identical and `instanceof` blind to it.
|
||||
* Rebuilding the nodes as plain objects reproduces that from inside a single
|
||||
* process, so anything that reads zod's internals stays pinned to `def.type`.
|
||||
*/
|
||||
|
||||
import type { z } from 'zod';
|
||||
|
||||
/** Strips the prototype off a schema node and everything it wraps. */
|
||||
function strip(node: unknown): unknown {
|
||||
const def = { ...(node as { def: Record<string, unknown> }).def };
|
||||
if (def.innerType) def.innerType = strip(def.innerType);
|
||||
return { def };
|
||||
}
|
||||
|
||||
export function foreignZodSchema<S extends z.ZodObject<any>>(schema: S): S {
|
||||
return {
|
||||
// Parsing is not what is under test — delegate it and keep the real
|
||||
// behaviour, so only the introspection path sees the foreign nodes.
|
||||
safeParse: (value: unknown) => schema.safeParse(value),
|
||||
shape: Object.fromEntries(
|
||||
Object.entries(schema.shape).map(([key, node]) => [key, strip(node)])
|
||||
),
|
||||
} as unknown as S;
|
||||
}
|
||||
@@ -5,12 +5,39 @@
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
import { Cube, Manifest } from '../src/cubes/types.js';
|
||||
import { Cube, Manifest } from '../src/types.js';
|
||||
import { foreignZodSchema } from './helpers/foreign-zod.js';
|
||||
|
||||
const cube = (schema: z.ZodObject<any>) =>
|
||||
new Cube(Manifest.create({ id: 'c', name: 'C', schema }), '/cubes/c', 'deploy.py');
|
||||
|
||||
describe('Cube', () => {
|
||||
it('reads id and name off the manifest', () => {
|
||||
const c = cube(z.object({}));
|
||||
|
||||
expect(c.id).toBe('c');
|
||||
expect(c.name).toBe('C');
|
||||
});
|
||||
|
||||
it('defaults its source to its own directory', () => {
|
||||
// What the loader overrides when a cube arrives from a package; a cube
|
||||
// built by hand still has to answer the question.
|
||||
expect(cube(z.object({})).source).toEqual({ type: 'dir', dir: '/cubes/c' });
|
||||
});
|
||||
|
||||
it('keeps the source it was constructed with', () => {
|
||||
const source = { type: 'package' as const, packageName: '@acme/cubes-net', dir: '/pkg/cubes' };
|
||||
const c = new Cube(
|
||||
Manifest.create({ id: 'c', name: 'C' }),
|
||||
'/pkg/cubes/c',
|
||||
'deploy.py',
|
||||
source
|
||||
);
|
||||
|
||||
expect(c.source).toBe(source);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cube.getDefaults', () => {
|
||||
it('resolves every default when the whole schema parses', () => {
|
||||
const c = cube(
|
||||
@@ -109,3 +136,36 @@ describe('Cube.requiredKeys', () => {
|
||||
expect(c.requiredKeys()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cube.secrets', () => {
|
||||
it('is empty when the manifest declares none', () => {
|
||||
const c = cube(z.object({ PASSWORD: z.string().default('x') }));
|
||||
|
||||
expect(c.secrets).toEqual([]);
|
||||
// No name-based guessing: only what the manifest says.
|
||||
expect(c.isSecret('PASSWORD')).toBe(false);
|
||||
});
|
||||
|
||||
it('reports what the manifest declared', () => {
|
||||
const c = new Cube(
|
||||
Manifest.create({
|
||||
id: 'c',
|
||||
name: 'C',
|
||||
schema: z.object({ USER: z.string(), PASSWORD: z.string() }),
|
||||
secrets: ['PASSWORD'],
|
||||
}),
|
||||
'/cubes/c',
|
||||
'deploy.py'
|
||||
);
|
||||
|
||||
expect(c.secrets).toEqual(['PASSWORD']);
|
||||
expect(c.isSecret('PASSWORD')).toBe(true);
|
||||
expect(c.isSecret('USER')).toBe(false);
|
||||
});
|
||||
|
||||
it('defaults to an empty list on a manifest built by hand', () => {
|
||||
const c = new Cube({ id: 'c', name: 'C', schema: z.object({}) }, '/cubes/c', 'deploy.py');
|
||||
|
||||
expect(c.secrets).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Tests for cubes/utils module
|
||||
* Tests for the uniqid helper
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { uniqid } from '../src/cubes/utils.js';
|
||||
import { uniqid } from '../src/utils.js';
|
||||
|
||||
describe('uniqid', () => {
|
||||
it('generates string of default length (5)', () => {
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": ".tsbuildinfo",
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"lib": ["ES2020"],
|
||||
"composite": true,
|
||||
"module": "NodeNext",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["coverage", "node_modules", "dist"],
|
||||
"references": []
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['src/**/*.test.ts', 'tests/**/*.test.ts'],
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'json-summary', 'html'],
|
||||
include: ['src/**/*.ts'],
|
||||
exclude: [
|
||||
'src/**/*.test.ts',
|
||||
// Pure re-export barrel: no logic to cover.
|
||||
'src/index.ts',
|
||||
],
|
||||
thresholds: {
|
||||
branches: 85,
|
||||
functions: 85,
|
||||
lines: 80,
|
||||
statements: 80,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
+112
-17
@@ -33,7 +33,7 @@ Nopy wraps pyinfra with structure, validation, and an interactive experience for
|
||||
|
||||
A cube is a **directory** containing two files:
|
||||
|
||||
- **JavaScript manifest**: `manifest.mjs` defining schema, dependencies, defaults, and hooks
|
||||
- **JavaScript manifest**: `manifest.mjs` defining schema, dependencies, defaults, secrets, and hooks
|
||||
- **Python deployment script**: `deploy.py`, a plain pyinfra script
|
||||
|
||||
Configuration variables are declared in the manifest and validated with Zod schemas before the deployment script runs.
|
||||
@@ -96,7 +96,7 @@ apt.packages(
|
||||
)
|
||||
```
|
||||
|
||||
Every key defined in the manifest `schema` is guaranteed to be present on `host.data` — either from the Zod `.default()`, from `.nopyrc.json`, from a dependency, or from a user prompt.
|
||||
Every key defined in the manifest `schema` is guaranteed to be present on `host.data` — either from the Zod `.default()`, from `.nopyrc.json`, from a recorded session, from a dependency, or from a user prompt.
|
||||
|
||||
**Value types**: pyinfra parses `--data` values before your script sees them. `"true"` / `"false"` become booleans, numeric strings become `int`, valid JSON becomes the parsed structure, and everything else stays a string. This is why `UPDATE` can be handed straight to pyinfra's `update=` argument, while `PACKAGES` is wrapped in `str(...)` before splitting.
|
||||
|
||||
@@ -104,18 +104,57 @@ Every key defined in the manifest `schema` is guaranteed to be present on `host.
|
||||
|
||||
Variable defaults are defined directly in the Zod schema using `.default()`. This ensures that every cube has a predictable starting state and provides type-safe default values.
|
||||
|
||||
**Priority order (lowest to highest):**
|
||||
A variable can be set from several places in one run. Every assignment is kept, tagged with where it came from — its **origin** — and the highest-ranked origin wins.
|
||||
|
||||
1. Zod schema `.default()` values
|
||||
2. Global `env` from `.nopyrc.json`
|
||||
3. User prompts, or the recorded answers on session replay
|
||||
4. Variables passed in by a dependency or a hook
|
||||
**Origins, lowest to highest:**
|
||||
|
||||
| Origin | Set by |
|
||||
| --------- | ------------------------------------------------------- |
|
||||
| `default` | the Zod schema's `.default()` |
|
||||
| `env` | the `env` block of `.nopyrc.json` |
|
||||
| `session` | a value recorded in a session file or history entry |
|
||||
| `prompt` | what the user typed |
|
||||
| `param` | a dependency spec or a `before`/`after` hook |
|
||||
|
||||
This allows cubes to ship with reasonable defaults while still allowing users to override them globally via `.nopyrc.json` or interactively during deployment. Because `env` outranks the schema, `.nopyrc.json` is also what steers a run started with `--use-defaults`, which never prompts.
|
||||
|
||||
3 and 4 rarely compete: a key a dependency supplies is left out of the prompt entirely, so the user is only ever asked about the keys nothing else has set.
|
||||
`prompt` and `param` rarely compete: a key a dependency supplies is left out of the prompt entirely, so the user is only ever asked about the keys nothing else has set.
|
||||
|
||||
A field declared without `.default()` has none of sources 1 and 2 to fall back on. It is prompted for like any other, with an empty initial value — but a run that cannot prompt (`--use-defaults`) fails on it unless `env` or a dependency provides it.
|
||||
Ranking by origin rather than by arrival order is what makes replay work: a recorded value is applied *before* the cube would be prompted for, and prompting can still override it, but a `--data` value pushed in by a dependency is never clobbered by a stale recording.
|
||||
|
||||
A field declared without `.default()` has no `default` origin to fall back on. It is prompted for like any other, with an empty initial value — but a run that cannot prompt (`--use-defaults`) fails on it unless `env` or a dependency provides it.
|
||||
|
||||
#### Secrets
|
||||
|
||||
A manifest can name schema keys that hold sensitive values:
|
||||
|
||||
```javascript
|
||||
export default cubes.Manifest({
|
||||
id: 'user:add',
|
||||
name: 'Add a user account',
|
||||
secrets: ['PASSWORD'],
|
||||
schema: z.object({
|
||||
USERNAME: z.string().describe('Username for the new account').default('deploy'),
|
||||
PASSWORD: z.string().describe('Password for the new user account').default('changeme'),
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
Every entry must be a key of `schema`; naming anything else is a manifest error and aborts the run, so a typo fails loudly instead of silently leaving a value unprotected.
|
||||
|
||||
Declaring a key a secret changes three things:
|
||||
|
||||
- **It is never written to a session file or to the history.** Everything else the run settled on is recorded — including values that came from a `.default()` — but declared secrets are left out.
|
||||
- **It is masked wherever a command or a plan is printed** — `--dry-run`, `--print-only`, and the debug log all show `********` in place of the value, in the variable list *and* in the `pyinfra` command line above it. The SSH password passed via `--password` is masked the same way, whether or not any cube declares secrets.
|
||||
- **It is re-prompted on replay**, since there is nothing recorded to replay from (see [Session Recording and Replay](#session-recording-and-replay)).
|
||||
|
||||
Nopy does not guess. A key called `PASSWORD` in a manifest that declares no `secrets` is treated as an ordinary variable — recorded, and printed in the clear.
|
||||
|
||||
Three limits are worth knowing, because `secrets` keeps a value out of the files nopy writes and nothing more:
|
||||
|
||||
- **It is on the command line.** pyinfra takes its data as `--data KEY=value`, so the real value is visible in `ps` for as long as the deployment runs. Masking covers nopy's own output, not the process table.
|
||||
- **The prompt shows it.** The variable form displays and pre-fills what it is asking about, so a secret is on screen while it is being entered or confirmed.
|
||||
- **A `.default()` is not protected.** A default lives in the manifest, in plain text, wherever the manifest is checked in. Give a secret a placeholder default like `changeme` if it needs one at all, never a real credential.
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -125,6 +164,7 @@ Uses `.nopyrc.json` files (project-level or home directory) containing:
|
||||
{
|
||||
"hosts": ["host1.example.com", "host2.example.com"],
|
||||
"cubeDirs": ["./cubes", "../shared-cubes"],
|
||||
"cubePackages": ["@bitsquare/cubes-core"],
|
||||
"env": {
|
||||
"SHARED_VAR": "value"
|
||||
},
|
||||
@@ -144,6 +184,8 @@ Uses `.nopyrc.json` files (project-level or home directory) containing:
|
||||
|
||||
`history` controls automatic session recording (see [Deployment History](#deployment-history)), and `execution.continueOnError` sets the default for `--continue-on-error`.
|
||||
|
||||
`cubeDirs` holds paths, `cubePackages` holds installed npm packages that ship cubes — see [Cube Discovery](#cube-discovery) below and [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md) for publishing your own. Both are additive, and both resolve relative to the config file that named them, not to the working directory: a `.nopyrc.json` two levels up may name a package that only exists in *its* `node_modules`.
|
||||
|
||||
#### Logging Configuration
|
||||
|
||||
Control pyinfra output verbosity and debug information using the `log` configuration object:
|
||||
@@ -213,12 +255,16 @@ Sessions are stored in `.nopysession.json` files with the following structure:
|
||||
|
||||
**Structure Details:**
|
||||
|
||||
- **`cubes`**: Array of cubes with only cube-specific variables (not global env vars)
|
||||
- **`env`**: Global environment variables shared across cubes (like in `.nopyrc.json`)
|
||||
- **`cubes`**: Array of cubes with the variable values that cube ran with
|
||||
- **`env`**: The `env` block of `.nopyrc.json` as it stood at record time, kept for reference
|
||||
- **`hosts`**: Array of target hosts
|
||||
- **`auth`**: Authentication configuration (passwords are never stored)
|
||||
|
||||
**Security Note**: Passwords are never stored in session files. If a session uses password authentication, you'll be prompted for the password during replay.
|
||||
**What is recorded:** every value each cube settled on, regardless of where it came from — a value the user typed, one inherited from `.nopyrc.json` `env`, one a dependency supplied, and one that fell through to the schema's `.default()` are all written out the same way. A session is therefore a full snapshot rather than a diff, and a `--use-defaults` run produces a session with real values in it instead of an empty one.
|
||||
|
||||
The consequence is that replay is faithful rather than re-derived: the recorded value outranks the current `.nopyrc.json` `env` and the current schema default, so editing either one does not silently change what a replay does. To pick up a new default, record a fresh session.
|
||||
|
||||
**Security Note**: Passwords are never stored in session files. This covers both the SSH password — a session records the auth *method* and username, never the credential — and any schema key a cube's manifest lists under [`secrets`](#secrets). Both are re-prompted on replay.
|
||||
|
||||
#### Recording a Session
|
||||
|
||||
@@ -240,12 +286,48 @@ nopy install --load-session my-deployment.nopysession.json
|
||||
# Only password authentication will prompt for credentials
|
||||
```
|
||||
|
||||
A replay runs straight through without asking anything, with three exceptions. Password authentication always re-prompts. A session with no recorded host falls back to the host picker. And a cube is re-prompted for its declared secrets, plus for any required variable the session has no value for — which happens when the cube's schema has gained a field since the session was written.
|
||||
|
||||
Those re-prompts are what a session cannot supply, so `--use-defaults` cannot paper over them: combining `-D` with a replay that needs either fails with a message naming the keys rather than deploying with a placeholder. Put the values under `env` in `.nopyrc.json` to make such a replay unattended.
|
||||
|
||||
### Cube Discovery
|
||||
|
||||
Nopy searches for cubes in:
|
||||
|
||||
1. Directories specified in `.nopyrc.json` `cubeDirs`
|
||||
2. Directories containing a `.npcubes` marker file (searching upwards from current directory)
|
||||
2. Cube directories of every package listed in `.nopyrc.json` `cubePackages`
|
||||
3. Directories containing a `.npcubes` marker file (searching upwards from current directory)
|
||||
|
||||
All three are unioned and scanned the same way. A directory is a cube when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a deploy script (`deploy.py` or `*.deploy.py`); dotted directories and `node_modules` are skipped during the scan.
|
||||
|
||||
#### Cube packages
|
||||
|
||||
A cube package is an ordinary npm package that ships cube directories and points at them from its own `package.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@bitsquare/cubes-core",
|
||||
"nopy": { "cubes": ["./cubes"] }
|
||||
}
|
||||
```
|
||||
|
||||
Install it and name it — nothing needs linking or copying:
|
||||
|
||||
```sh
|
||||
pnpm add -D @bitsquare/cubes-core
|
||||
```
|
||||
|
||||
```json
|
||||
{ "cubePackages": ["@bitsquare/cubes-core"] }
|
||||
```
|
||||
|
||||
Naming a package is a statement that cubes are expected from it, so anything wrong is an error that aborts the run rather than a silent skip: the package is not installed, it declares no `nopy.cubes`, or an entry points at a directory that does not exist or lies outside the package.
|
||||
|
||||
#### Ids are claimed globally
|
||||
|
||||
A cube id such as `apt:essentials` is claimed across every source at once, not per directory or per package. Two cubes with the same id abort the run with an error naming both and where each came from. There is no precedence rule and no shadowing — a local cube does not quietly win over a packaged one, in either direction. Prefix your own cubes distinctly if you point `cubeDirs` at a local tree alongside an installed bundle.
|
||||
|
||||
Writing cubes to publish is covered in [CUBE-BUNDLES.md](docs/CUBE-BUNDLES.md).
|
||||
|
||||
## Command Line Usage
|
||||
|
||||
@@ -319,6 +401,17 @@ have no default values. Set them under "env" in .nopyrc.json, pass them from a
|
||||
dependency, or drop --use-defaults to be prompted.
|
||||
```
|
||||
|
||||
Pairing `-D` with a replay fails the same way when the replay would have to ask
|
||||
something — a declared [secret](#secrets), which is never recorded, or a required
|
||||
variable the session has no value for. Both are the sources `-D` has no substitute
|
||||
for, so it stops rather than deploying a placeholder:
|
||||
|
||||
```
|
||||
Error: Cube "user:add" cannot be replayed with --use-defaults: PASSWORD would
|
||||
have to be entered. Secrets are never recorded in a session. Replay without
|
||||
--use-defaults, or set the values under "env" in .nopyrc.json.
|
||||
```
|
||||
|
||||
**Use SSH key authentication**:
|
||||
|
||||
```bash
|
||||
@@ -434,20 +527,20 @@ Session History:
|
||||
Total: 2 session(s)
|
||||
```
|
||||
|
||||
Each entry records the selected cubes together with the variable values that were answered at the prompts, the target hosts, the authentication method, and the username — never the password. Pass an ID to `-H` to run that exact combination again:
|
||||
Each entry records the selected cubes together with every variable value they ran with, the target hosts, the authentication method, and the username — never the password, and never a key the manifest declared a [secret](#secrets). Pass an ID to `-H` to run that exact combination again:
|
||||
|
||||
```bash
|
||||
nopy install -H mdk0zzp8b71cq
|
||||
```
|
||||
|
||||
A replay is non-interactive: cube selection, host, and variable values all come from the entry, so nopy runs straight through without asking anything. The two exceptions are password authentication, which always re-prompts, and an entry with no recorded host, which falls back to the host picker.
|
||||
A replay is non-interactive: cube selection, host, and variable values all come from the entry, so nopy runs straight through without asking anything. It asks only for what the entry cannot hold — the password under password authentication, and any declared secret — plus the host picker when the entry recorded none.
|
||||
|
||||
Two things are worth knowing before relying on an older entry:
|
||||
|
||||
- **Recorded values are applied as defaults, not as a frozen snapshot.** If a cube's schema has gained a variable since the run was recorded, the replay neither prompts for it nor fails — the new variable quietly takes its Zod `.default()`. Global `env` values are likewise read from the *current* `.nopyrc.json` rather than from the entry.
|
||||
- **Recorded values win over the current configuration.** The entry is a snapshot of everything the run settled on, so editing a cube's `.default()` or the `env` block of `.nopyrc.json` afterwards does not change what the replay does. A variable the schema has gained *since* the entry was written has nothing recorded: if it has a `.default()` the replay quietly takes it, and if it is required the replay prompts for it.
|
||||
- **A replay fails if a cube no longer exists.** Renaming or deleting a cube id makes every history entry that referenced it unreplayable: nopy logs `Cube from session not found` and then aborts with `Cube not found: <id>`.
|
||||
|
||||
The history lives in `.nopy.history.json` in the working directory and uses the same structure as a session file, so trimming the array by hand is a perfectly good way to prune it. It does contain the variable values that were entered, which is why it is listed in this repository's `.gitignore` — treat it like any other file holding deployment configuration. A corrupt or unreadable history file is treated as empty rather than raising an error, which looks exactly like a project that has never been deployed from.
|
||||
The history lives in `.nopy.history.json` in the working directory and uses the same structure as a session file, so trimming the array by hand is a perfectly good way to prune it. It does contain the variable values a run used, which is why it is listed in this repository's `.gitignore` — treat it like any other file holding deployment configuration. A corrupt or unreadable history file is treated as empty rather than raising an error, which looks exactly like a project that has never been deployed from.
|
||||
|
||||
For a run you want to keep indefinitely, don't rely on history — it rotates. Use `--save-session` to write it to a file you control (see [Session Recording and Replay](#session-recording-and-replay)).
|
||||
|
||||
@@ -468,7 +561,9 @@ npm run debug
|
||||
## Documentation
|
||||
|
||||
- [Cube Hooks](docs/HOOKS.md) - Lifecycle hooks for dynamic orchestration
|
||||
- [Cube Bundles](docs/CUBE-BUNDLES.md) - Distributing cubes as npm packages
|
||||
- [Session Format](docs/SESSION_FORMAT.md) - Internal JSON/MJS session structure
|
||||
- [API Reference](docs/API.md) - Types and exported functions
|
||||
|
||||
## Resources
|
||||
|
||||
|
||||
@@ -65,6 +65,20 @@ interface NopyResult {
|
||||
|
||||
The cubes module provides types and functions for working with deployment units.
|
||||
|
||||
The authoring half of it — `Manifest`, `Cube`, `Hook`, `uniqid` and the rest —
|
||||
actually lives in **[`@bitsquare/nopy-cube`](../../nopy-cube)**, a package with
|
||||
no CLI and no dependency other than zod. `@bitsquare/nopy` re-exports all of it,
|
||||
so both of these work:
|
||||
|
||||
```javascript
|
||||
import { Manifest } from '@bitsquare/nopy-cube'; // in a manifest.mjs — prefer this
|
||||
import { cubes } from '@bitsquare/nopy'; // cubes.Manifest — still supported
|
||||
```
|
||||
|
||||
Import from `nopy-cube` in a cube bundle you intend to publish: it lets the
|
||||
bundle depend on the authoring types without pulling the whole CLI in as a
|
||||
dependency. See [CUBE-BUNDLES.md](CUBE-BUNDLES.md).
|
||||
|
||||
### Types
|
||||
|
||||
#### `Cube<Schema>`
|
||||
@@ -76,6 +90,7 @@ interface Cube<Schema extends z.AnyZodObject = z.AnyZodObject> {
|
||||
key: string; // Unique identifier
|
||||
name: string; // Human-readable name
|
||||
dir: string; // Absolute path to cube directory
|
||||
source: CubeSource; // Where it was discovered
|
||||
dependencies: string[];
|
||||
schema: Schema;
|
||||
defaults: () => z.infer<Schema>;
|
||||
@@ -84,6 +99,18 @@ interface Cube<Schema extends z.AnyZodObject = z.AnyZodObject> {
|
||||
}
|
||||
```
|
||||
|
||||
#### `CubeSource`
|
||||
|
||||
Where a cube came from. Carried so that a duplicate-id error can name the origin
|
||||
of each claimant, which is the difference between a usable error message and a
|
||||
puzzle when the collision is between a local tree and an installed bundle.
|
||||
|
||||
```typescript
|
||||
type CubeSource =
|
||||
| { type: 'dir'; dir: string }
|
||||
| { type: 'package'; packageName: string; dir: string };
|
||||
```
|
||||
|
||||
#### `Manifest<Schema>`
|
||||
|
||||
Cube manifest (used in `manifest.mjs` files).
|
||||
@@ -124,7 +151,9 @@ interface HookContext {
|
||||
|
||||
#### `loadCubes()`
|
||||
|
||||
Loads all cubes from discovered cube directories.
|
||||
Loads all cubes from discovered cube directories — `cubeDirs`, the directories
|
||||
declared by every package in `cubePackages`, and any ancestor directory holding a
|
||||
`.npcubes` marker.
|
||||
|
||||
```typescript
|
||||
const { cubes, errors } = await loadCubes();
|
||||
@@ -139,6 +168,27 @@ interface LoadResult {
|
||||
}
|
||||
```
|
||||
|
||||
`errors` is non-empty for a duplicate id, a manifest that fails to load, a
|
||||
package in `cubePackages` that is not installed or declares no cubes, and a
|
||||
`nopy.cubes` entry that is missing or points outside its package. Any of them
|
||||
aborts the run — none is a silent skip.
|
||||
|
||||
#### `resolveCubePackages(refs)`
|
||||
|
||||
Resolves `CubePackageRef[]` to installed packages and their cube directories.
|
||||
Called by `loadCubes()`; exported because the resolution failures are worth
|
||||
testing on their own.
|
||||
|
||||
```typescript
|
||||
const { packages, errors } = resolveCubePackages(config.cubePackages);
|
||||
|
||||
interface CubePackage {
|
||||
name: string; // the name it was requested under
|
||||
root: string; // absolute path to the package root
|
||||
dirs: string[]; // absolute paths from its `nopy.cubes` field
|
||||
}
|
||||
```
|
||||
|
||||
#### `resolveDependencies(cubes, selectedCubeNames)`
|
||||
|
||||
Resolves all transitive dependencies for selected cubes.
|
||||
@@ -452,11 +502,31 @@ Configuration file structure.
|
||||
interface NopyConfig {
|
||||
hosts: string[];
|
||||
cubeDirs: string[];
|
||||
cubePackages: CubePackageRef[];
|
||||
env: EnvConfig;
|
||||
log?: LogConfig;
|
||||
}
|
||||
```
|
||||
|
||||
#### `CubePackageRef`
|
||||
|
||||
A package named in `cubePackages`, paired with where it was named. In the config
|
||||
file an entry is just a string (`"@bitsquare/cubes-core"`); `loadConfig()`
|
||||
normalises it.
|
||||
|
||||
```typescript
|
||||
interface CubePackageRef {
|
||||
/** The package name, as written in the config. */
|
||||
spec: string;
|
||||
/** Directory of the config file that named it — resolution starts here. */
|
||||
from: string;
|
||||
}
|
||||
```
|
||||
|
||||
`from` is what makes a package named in a parent config resolve against *that*
|
||||
config's `node_modules`, not the working directory's. It is the same problem
|
||||
`PATH_PROPERTIES` solves for relative `cubeDirs`.
|
||||
|
||||
#### `LogConfig`
|
||||
|
||||
Logging configuration.
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
# Cube bundles
|
||||
|
||||
How to package cubes as an npm package so other projects can install them, and
|
||||
what changes once a cube lives in `node_modules` instead of in your own tree.
|
||||
|
||||
If you only want to *use* a published bundle, you need one line of config:
|
||||
|
||||
```json
|
||||
{ "cubePackages": ["@bitsquare/cubes-core"] }
|
||||
```
|
||||
|
||||
The rest of this document is for writing one.
|
||||
|
||||
- [What a bundle is](#what-a-bundle-is)
|
||||
- [The package manifest](#the-package-manifest)
|
||||
- [Writing the cubes](#writing-the-cubes)
|
||||
- [Ids are claimed globally](#ids-are-claimed-globally)
|
||||
- [An installed bundle is read-only](#an-installed-bundle-is-read-only)
|
||||
- [How resolution actually works](#how-resolution-actually-works)
|
||||
- [Publishing](#publishing)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
## What a bundle is
|
||||
|
||||
An ordinary npm package that ships cube directories and points at them from its
|
||||
own `package.json`. There is no build step, no plugin API and no entry point —
|
||||
nopy reads the directories off disk and imports each `manifest.mjs` directly.
|
||||
|
||||
```
|
||||
@acme/cubes-web
|
||||
├── package.json nopy.cubes → ["./cubes"]
|
||||
├── README.md
|
||||
└── cubes/
|
||||
├── nginx/
|
||||
│ ├── manifest.mjs
|
||||
│ └── deploy.py
|
||||
└── certbot/
|
||||
├── manifest.mjs
|
||||
└── deploy.py
|
||||
```
|
||||
|
||||
`@bitsquare/cubes-core` in this repository is the worked example, and is consumed
|
||||
by this repository through exactly the mechanism described here — it is not
|
||||
special-cased.
|
||||
|
||||
## The package manifest
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@acme/cubes-web",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"nopy": { "cubes": ["./cubes"] },
|
||||
"files": ["cubes", "!cubes/**/*.log", "README.md", "LICENSE"],
|
||||
"publishConfig": { "access": "public" },
|
||||
"dependencies": {
|
||||
"@bitsquare/nopy-cube": "^1.0.0",
|
||||
"zod": "^4.4.3"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**`nopy.cubes`** is the only field nopy requires. It is an array of directories,
|
||||
relative to the package root, each scanned recursively for cubes. Several
|
||||
entries are fine; a single `["./cubes"]` is the norm. Every entry must exist and
|
||||
must stay inside the package — a path escaping the root is refused, not resolved.
|
||||
|
||||
**`type: "module"`** matters: manifests are ESM. Without it a `manifest.mjs` still
|
||||
loads (the extension carries the day), but anything it imports relatively will
|
||||
not behave the way you expect.
|
||||
|
||||
**`files`** decides the tarball. Note the negation: a cube that has been run
|
||||
leaves a `pyinfra-debug.log` next to its `deploy.py`, and `.gitignore` has no
|
||||
effect on what npm packs. Check with `npm pack --dry-run` before publishing.
|
||||
|
||||
**Dependencies** are `@bitsquare/nopy-cube` and `zod`, both real dependencies
|
||||
rather than peers — a bundle is a leaf, and the copies it gets are the copies its
|
||||
manifests use. Do **not** depend on `@bitsquare/nopy`: the CLI is what installs
|
||||
your bundle, not the other way round, and depending on it invites two copies of
|
||||
the same code into one tree.
|
||||
|
||||
## Writing the cubes
|
||||
|
||||
A cube directory holds a manifest (`manifest.mjs` or `*.manifest.mjs`) and a
|
||||
deploy script (`deploy.py` or `*.deploy.py`). Anything else in the directory is
|
||||
invisible to the loader but readable from the script, which runs with the cube
|
||||
directory as its working directory.
|
||||
|
||||
```javascript
|
||||
// cubes/nginx/manifest.mjs
|
||||
import { Manifest } from '@bitsquare/nopy-cube';
|
||||
import { z } from 'zod';
|
||||
|
||||
export default Manifest({
|
||||
id: 'web:nginx',
|
||||
name: 'Install and configure nginx',
|
||||
dependencies: () => ['apt:essentials'],
|
||||
secrets: ['TLS_KEY'],
|
||||
schema: z.object({
|
||||
SERVER_NAME: z.string().describe('Server name').default('example.com'),
|
||||
TLS_KEY: z.string().describe('TLS private key (PEM)').default(''),
|
||||
HTTP2: z.boolean().describe('Enable HTTP/2').default(true),
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
||||
```python
|
||||
# cubes/nginx/deploy.py
|
||||
from pyinfra import host
|
||||
from pyinfra.operations import apt, files
|
||||
|
||||
SERVER_NAME = host.data.SERVER_NAME
|
||||
|
||||
apt.packages(name='Install nginx', packages=['nginx'], _sudo=True)
|
||||
```
|
||||
|
||||
Import **`@bitsquare/nopy-cube`**, not `@bitsquare/nopy`. It is types and a
|
||||
factory with zod as its only peer — no CLI, no prompts, no process spawning — so
|
||||
your bundle stays a leaf. (`@bitsquare/nopy` re-exports the same surface as
|
||||
`cubes.Manifest`, which is what older manifests use. It still works; it just
|
||||
drags the CLI into your dependency graph if you declare it.)
|
||||
|
||||
Four things the schema is load-bearing for:
|
||||
|
||||
- **`.describe()` is the prompt label.** A field without one prompts with its raw
|
||||
key.
|
||||
- **`.default()` makes the field optional.** A field with no default is required,
|
||||
and is re-prompted on replay if a session has no value for it.
|
||||
- **Every schema key reaches pyinfra** as `--data KEY=value`, so `host.data.KEY`
|
||||
is always defined. pyinfra parses the values itself: `"true"` arrives as a
|
||||
bool, `"8080"` as an int.
|
||||
- **`secrets` names keys whose values must not be persisted.** They are excluded
|
||||
from session files and history, masked wherever a command is printed, and
|
||||
re-prompted on replay. Naming a key that is not in the schema is a load error.
|
||||
A secret is still visible in `ps` while pyinfra runs — masking covers nopy's
|
||||
own output, not the process table — so treat it as protection against writing
|
||||
credentials to disk, not as protection against a shared host.
|
||||
|
||||
`dependencies` is a function of the *collected* variables, so it can branch on
|
||||
what the user actually answered, and it may pass parameters:
|
||||
|
||||
```javascript
|
||||
dependencies: (v) => (v.HTTP2 ? ['apt:essentials', ['web:tls', { MODE: 'strict' }]] : []),
|
||||
```
|
||||
|
||||
`before` / `after` hooks get a context whose `exec(id, vars)` pulls in any cube
|
||||
by id, declared dependency or not. See [HOOKS.md](HOOKS.md).
|
||||
|
||||
## Ids are claimed globally
|
||||
|
||||
An id is claimed across every source at once — `cubeDirs`, `.npcubes` trees and
|
||||
every installed bundle share one flat namespace. Two cubes claiming the same id
|
||||
abort the run with an error naming both and where each came from.
|
||||
|
||||
There is no precedence and no shadowing, deliberately, in either direction: a
|
||||
local cube does not quietly win over a packaged one, and installing a second
|
||||
bundle cannot silently change what an existing id deploys. Overriding a cube from
|
||||
a bundle is not a supported operation; fork the cube under your own id instead.
|
||||
|
||||
So prefix distinctly. `@acme/cubes-web` claiming `nginx` is asking for trouble the
|
||||
first time someone installs a second bundle; `web:nginx` is not. Ids need not
|
||||
mirror the directory layout — `cubes/network/tailscale` declares `net:tailscale`
|
||||
— so the prefix is free.
|
||||
|
||||
An id is also the session key. Renaming one silently invalidates every recorded
|
||||
session that used it, so treat a rename as a breaking change of the bundle.
|
||||
|
||||
## An installed bundle is read-only
|
||||
|
||||
Under pnpm, installed files are **hardlinked into a global store shared by every
|
||||
project on the machine**. A cube that writes next to its own `deploy.py` does not
|
||||
just dirty one `node_modules` — it corrupts that store for every other project.
|
||||
|
||||
Write to `/tmp`, to a path the user configured, or to the remote host. Never to
|
||||
the cube's own directory. Files the cube needs to *read* (templates, config
|
||||
fragments, systemd units) are fine and are exactly what the cube directory is for
|
||||
— `deploy.py` runs with it as the working directory, so `files.template('nginx.conf.j2', ...)`
|
||||
resolves.
|
||||
|
||||
This is the one constraint that does not exist while the cubes live in your own
|
||||
repo, which makes it the one most likely to be discovered late. Test against an
|
||||
installed copy, not a linked one.
|
||||
|
||||
## How resolution actually works
|
||||
|
||||
Worth knowing, because two of the failure modes are otherwise baffling.
|
||||
|
||||
**Where a package is looked up from.** Each `cubePackages` entry is resolved from
|
||||
the directory of the config file that named it, not from the working directory.
|
||||
Configs merge upward, so a `.nopyrc.json` two levels up can name a bundle that
|
||||
only exists in *its* `node_modules`, and it resolves. The lookup reads
|
||||
`package.json` off disk via `createRequire(...).resolve.paths()` rather than
|
||||
going through `exports` — a bundle ships directories and has no entry point to
|
||||
declare.
|
||||
|
||||
**Why the loader does not simply scan `node_modules`.** It cannot: pnpm plants a
|
||||
symlink at `node_modules/<name>`, and `readdir` reports it as a symlink, not a
|
||||
directory, so a recursive scan skips every package silently. Naming packages
|
||||
explicitly is the fix, and it is also the reason `node_modules` is skipped during
|
||||
the cube scan itself.
|
||||
|
||||
**How a manifest finds its imports.** Ordinary Node resolution, from the
|
||||
manifest's own directory. An installed bundle has its own `node_modules` with
|
||||
`@bitsquare/nopy-cube` and `zod` in it, so this just works. A hand-written cube
|
||||
sitting in a directory with no `node_modules` would historically fail with
|
||||
`ERR_MODULE_NOT_FOUND`; nopy now registers a resolve hook that catches exactly
|
||||
that case and falls back to resolving `@bitsquare/nopy-cube`, `@bitsquare/nopy`
|
||||
and `zod` from the running CLI. Normal resolution is always tried first, so a
|
||||
cube that ships its own zod keeps it. Treat the hook as a convenience for local
|
||||
cubes — a published bundle must declare its dependencies properly.
|
||||
|
||||
**Two copies of zod is a real hazard.** `instanceof` comparisons fail across
|
||||
copies, which is why nopy inspects schemas structurally (`schema.def.type`) and
|
||||
why `secrets` is a plain array rather than `.meta()` metadata — zod's metadata
|
||||
registry is per-copy, and a marker written into one copy's registry is invisible
|
||||
to another. Keep your zod range compatible with the CLI's (`^4.4.3`) and the
|
||||
package manager will usually give you one copy.
|
||||
|
||||
## Publishing
|
||||
|
||||
Nothing bundle-specific: `npm publish` (or `pnpm publish`) with a version bump.
|
||||
Some things worth deciding once:
|
||||
|
||||
- **Version the bundle independently of nopy.** There is no compatibility check
|
||||
between the two — the loader reads whatever `nopy.cubes` points at. Document
|
||||
the nopy version you test against in your README.
|
||||
- **Renaming or removing an id is breaking.** It invalidates recorded sessions
|
||||
and breaks any manifest listing it as a dependency, including manifests in
|
||||
other people's bundles.
|
||||
- **Changing a schema key is breaking** in the same way; adding one with a
|
||||
`.default()` is not.
|
||||
- **Test the installed shape, not the linked one.** `npm pack`, install the
|
||||
tarball into a throwaway directory with a `.nopyrc.json` naming it, and deploy
|
||||
from it. This is what catches a missing file, a cube that writes to its own
|
||||
directory, and an undeclared dependency — none of which show up while the
|
||||
package is symlinked into the repo that wrote it.
|
||||
|
||||
For an unattended check, replay a session file rather than reaching for `-P`
|
||||
alone, which still opens the interactive picker:
|
||||
|
||||
```sh
|
||||
nopy install -l session.json -P -D
|
||||
```
|
||||
|
||||
Note that a replay re-prompts for anything a manifest lists in `secrets` —
|
||||
those are never written to a session — so pick a cube without them, or put the
|
||||
values under `env` in `.nopyrc.json`.
|
||||
|
||||
For how this repository releases its own packages, see
|
||||
[README.PUBLISH.md](../../../README.PUBLISH.md).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Cause |
|
||||
| --- | --- |
|
||||
| `Cube package 'X' is not installed (looked up from …)` | Not installed, or installed somewhere other than the config that named it. The path in the message is where the lookup started. |
|
||||
| `Cube package 'X' declares no cubes` | Missing or malformed `nopy.cubes` in the package's `package.json`. It must be a non-empty array of strings. |
|
||||
| `'./cubes' does not exist in …` | The directory was not packed. Check `files` and `npm pack --dry-run`. |
|
||||
| `'…' points outside the package` | A `nopy.cubes` entry escaping the package root. Not allowed. |
|
||||
| `Duplicate cube id 'X' from N sources:` | Two or more cubes claiming one id; the message lists each source. Rename one — there is no precedence rule to lean on. |
|
||||
| `ERR_MODULE_NOT_FOUND` for `zod` or `@bitsquare/nopy-cube` | The bundle did not declare them as dependencies. The resolve-hook fallback covers loose local cubes, not published packages. |
|
||||
| `Invalid manifest in …: 'secrets' names X, which is not in the schema` | A `secrets` entry with no matching schema key — usually a typo or a renamed field. |
|
||||
| Cubes work linked, fail installed | Almost always a write into the cube's own directory, or a file missing from `files`. |
|
||||
@@ -1,6 +1,11 @@
|
||||
# Cube bundles as npm packages
|
||||
|
||||
Status: **Phase 0 has landed; Phases 1–6 are still a plan, not a record.**
|
||||
Status: **All six phases have landed. This document is now a record, not a plan.**
|
||||
The one thing still unproven is the publish lane against a real registry — see
|
||||
*Risks*.
|
||||
`cubePackages` resolves and loads end to end, `@bitsquare/nopy-cube` exists and
|
||||
the publish lane can ship a linked package. What is missing is a bundle to point
|
||||
`cubePackages` at.
|
||||
|
||||
Distributing cubes as npm packages so a project can `pnpm add @acme/cubes-net`
|
||||
and have its cubes show up in `nopy` alongside local ones.
|
||||
@@ -166,7 +171,7 @@ Rules:
|
||||
- A bundle must not ship a `.nopyrc.json`. Config discovery walks up from
|
||||
`process.cwd()`, never from cube directories, so it would never be read.
|
||||
|
||||
## Phase 2 — resolution
|
||||
## Phase 2 — resolution — **done**
|
||||
|
||||
### Config surface
|
||||
|
||||
@@ -271,7 +276,7 @@ it is exported from `src/cubes/index.ts` and covered by tests. The
|
||||
`node_modules` skip inside `scanDirectory` stays and is now *correct*: a
|
||||
bundle's own `node_modules` should not be scanned.
|
||||
|
||||
## Phase 3 — hard errors with attribution
|
||||
## Phase 3 — hard errors with attribution — **done**
|
||||
|
||||
`Cube` gains a source, as an optional fourth constructor parameter so the public
|
||||
signature stays backwards compatible:
|
||||
@@ -306,7 +311,7 @@ claim the same id they are mutually exclusive, and the fix is upstream.
|
||||
Surface the source in the interactive picker and in `--json` output so a user can
|
||||
see where a cube came from before running it.
|
||||
|
||||
## Phase 4 — `@bitsquare/nopy-cube`, the authoring package
|
||||
## Phase 4 — `@bitsquare/nopy-cube`, the authoring package — **done**
|
||||
|
||||
The problem: a manifest does `import { cubes } from '@bitsquare/nopy'`, resolved
|
||||
by ordinary Node resolution from the manifest's own directory. From inside
|
||||
@@ -350,20 +355,33 @@ already coverage-excluded barrels — so `import { cubes } from '@bitsquare/nopy
|
||||
in every existing manifest keeps working unchanged. Nothing in `cubes/` has to be
|
||||
touched at migration time.
|
||||
|
||||
Repo plumbing this requires:
|
||||
`cubes/types.ts` and `cubes/factories.ts` moved wholesale, with
|
||||
`tests/cubes.types.test.ts` and `tests/cubes.factories.test.ts` behind them.
|
||||
`tests/helpers/foreign-zod.ts` is duplicated rather than shared — fifteen lines,
|
||||
and the alternative is a test-only dependency edge between the packages.
|
||||
|
||||
- `tsconfig.base.json`: add `"@bitsquare/nopy-cube": ["./packages/nopy-cube/src"]`
|
||||
to `paths`.
|
||||
- Root `tsconfig.json`: add the project reference.
|
||||
- `packages/nopy/tsconfig.json`: `references` is currently `[]` — add
|
||||
`{ "path": "../nopy-cube" }`. This is the first reference edge in the repo, so
|
||||
`tsc --build` ordering starts mattering.
|
||||
Repo plumbing it took:
|
||||
|
||||
- `tsconfig.base.json`: `"@bitsquare/nopy-cube": ["./packages/nopy-cube/src"]`.
|
||||
- Root `tsconfig.json` and `packages/nopy/tsconfig.json`: the project reference.
|
||||
This is the first reference edge in the repo, and it broke the gate
|
||||
immediately: **`tsc --build --noEmit` is not legal once a project has
|
||||
references** — TS6310, "referenced project may not disable emit", because a
|
||||
composite project has to emit the declarations its dependents read. The root
|
||||
`typecheck` script is now plain `tsc --build`. It still fails on a type error,
|
||||
and it now also proves the build works; the cost is that it writes `dist`,
|
||||
which is gitignored.
|
||||
- `packages/nopy/package.json`: `"@bitsquare/nopy-cube": "workspace:*"`.
|
||||
- A `vitest.config.ts` for the new package with the same thresholds. `Manifest()`,
|
||||
`Manifest.create()` and `Cube.getDefaults()` all carry logic, so the relevant
|
||||
cases move over from `tests/cubes.factories.test.ts`.
|
||||
- `packages/nopy/vitest.config.ts`: a `resolve.alias` for `@bitsquare/nopy-cube`
|
||||
pointing at `../nopy-cube/src/index.ts`. Without it the workspace link
|
||||
resolves through `exports` to `dist`, so `pnpm test` on a clean checkout would
|
||||
fail until something had built it, and a stale `dist` would silently be what
|
||||
the tests ran against. The same config excludes `**/nopy-cube/**` from
|
||||
coverage — the aliased files were being counted against nopy's thresholds.
|
||||
- A `vitest.config.ts` for the new package with the same thresholds. It sits at
|
||||
100 % statements/functions/lines, 91 % branches.
|
||||
|
||||
### The release lane needs fixing first
|
||||
### The release lane needed fixing first
|
||||
|
||||
This is the part that is easy to miss. `link-workspace-packages` is unset and
|
||||
pnpm 10+ defaults it to `false`, so a plain semver range would resolve
|
||||
@@ -389,10 +407,40 @@ Pick one before publishing anything:
|
||||
passes — compute every snapshot version first, then publish — so `nopy` can pin
|
||||
the exact `nopy-cube` snapshot from the same run.
|
||||
|
||||
Recommendation: `pnpm publish`, and verify against the Gitea registry with a
|
||||
**Measured, both directions.** `npm pack` in `packages/nopy` produces a tarball
|
||||
whose manifest still reads `"@bitsquare/nopy-cube": "workspace:*"`; `pnpm pack`
|
||||
produces one that reads `"1.0.0-alpha0"`. So the failure was real and the fix
|
||||
works.
|
||||
|
||||
Went with `pnpm publish --ignore-scripts --no-git-checks` in both workflows.
|
||||
`--no-git-checks` is not optional in either: `release.yml` runs on a detached
|
||||
HEAD, and `publish-snapshot.yml` dirties the tree by stamping versions.
|
||||
(`pnpm pack` has no `--ignore-scripts`, only `pnpm publish` does.)
|
||||
|
||||
Three small scripts carry the parts that are easy to get wrong, all runnable
|
||||
locally:
|
||||
|
||||
- **`scripts/verify-pack.mjs`** — packs every publishable package and fails if a
|
||||
`workspace:` range survived into the tarball. Runs between build and publish
|
||||
in both workflows. Turns "npm would have shipped a broken manifest" from an
|
||||
install-time surprise into a red run.
|
||||
- **`scripts/publish-order.mjs`** — topologically sorts the publishable packages.
|
||||
`packages/*/` alphabetically puts `nopy` ahead of the `nopy-cube` it depends
|
||||
on; the snapshot workflow now iterates this instead.
|
||||
- **`scripts/linked-deps.mjs`** — lists a package's workspace links as
|
||||
`<name> <version>`, resolved by package name rather than by directory.
|
||||
`release.yml` uses it to refuse a release whose linked dependency is not on
|
||||
npmjs yet, which is the one mistake that cannot be taken back after 72 hours.
|
||||
|
||||
`publish-snapshot.yml` also became two passes over the packages: stamp every
|
||||
version first, then publish. `pnpm publish` substitutes the version the linked
|
||||
package declares *at pack time*, so `nopy-cube` has to be carrying its snapshot
|
||||
version before `nopy` is packed.
|
||||
|
||||
Still unverified: none of this has run against the Gitea registry. Worth a
|
||||
throwaway version before the first real release.
|
||||
|
||||
### Also: the resolve hook
|
||||
### Also: the resolve hook — built
|
||||
|
||||
Independent of the split, and worth building anyway — it retires the
|
||||
`ERR_MODULE_NOT_FOUND` gotcha CLAUDE.md documents for the local `cubes/` tree,
|
||||
@@ -414,33 +462,48 @@ Falling back for `zod` hands local cubes the *CLI's* zod instance, so no
|
||||
duplication arises there. Bundles are the case that duplicates it, and Phase 0.4
|
||||
is what makes that safe.
|
||||
|
||||
New `packages/nopy/src/nopy.resolve-hook.mjs`, registered once from `loadCubes()`
|
||||
`packages/nopy/src/cubes/resolve-hook.mjs`, registered once from `loadCubes()`
|
||||
before the first `import(manifestPath)`:
|
||||
|
||||
```ts
|
||||
module.register('./nopy.resolve-hook.mjs', import.meta.url, {
|
||||
data: { fallback: import.meta.resolve('./index.js') },
|
||||
});
|
||||
module.register('./resolve-hook.mjs', import.meta.url, { data: { from: import.meta.url } });
|
||||
```
|
||||
|
||||
The hook tries `next(specifier, ctx)` **first** and only falls back to the
|
||||
running CLI's own copy on failure. That ordering matters: a consumer that has its
|
||||
own `@bitsquare/nopy` installed keeps using it, so the hook never silently
|
||||
introduces version skew.
|
||||
`from` is a URL inside the running CLI's own package; the hook thread builds a
|
||||
`createRequire` from it and resolves the fallbacks out of the CLI's own
|
||||
dependencies.
|
||||
|
||||
Constraints:
|
||||
The hook tries `next(specifier, ctx)` **first** and only falls back on failure.
|
||||
That ordering matters: a consumer that has its own copy installed keeps using
|
||||
it, so the hook never silently introduces version skew. There is a test for
|
||||
exactly that — a stub `zod` beside the cube wins over the CLI's real one.
|
||||
|
||||
- `module.register()` is process-global and cannot be undone. Install it once,
|
||||
behind a module-level guard.
|
||||
Constraints, as built:
|
||||
|
||||
- `module.register()` is process-global and cannot be undone. Installed once,
|
||||
behind a module-level guard, and wrapped in a `try` — the hook is a
|
||||
convenience, so a registration failure must not abort a run.
|
||||
- The hook file runs on a separate thread; the `data` payload must be
|
||||
structured-cloneable (a string URL is).
|
||||
- The `.mjs` must ship in `dist` and be listed in `files` — it already is, via
|
||||
the `dist` entry.
|
||||
- It resolves `@bitsquare/nopy` and `zod`, not `@bitsquare/nopy-cube`. Bundles
|
||||
never depend on the hook; only the in-repo `cubes/` tree and hand-written local
|
||||
cubes do.
|
||||
- The `.mjs` has to reach `dist`, and `tsc` does not copy it: nopy's `build` is
|
||||
now `tsc && cp src/cubes/*.mjs dist/cubes/`. `files` already covers it via the
|
||||
`dist` entry.
|
||||
- It covers **three** specifiers, not the two the plan named: `zod`,
|
||||
`@bitsquare/nopy`, and `@bitsquare/nopy-cube` — a hand-written local cube is
|
||||
as entitled to the new authoring package as to the old one. Subpaths count
|
||||
(`@bitsquare/nopy/package.json`), anything else stays a hard failure.
|
||||
|
||||
## Phase 5 — proof of concept: `packages/cubes-core`
|
||||
**The tests have to spawn a real `node`.** Written inside the vitest worker they
|
||||
pass whether or not the hook is installed: vite resolves the dynamic import
|
||||
itself and finds `zod` from the project root. `tests/cubes.resolve-hook.test.ts`
|
||||
therefore runs each case in a child process, and the first case asserts the
|
||||
*failure* without the hook so the rest cannot silently stop proving anything.
|
||||
|
||||
**Verified end to end.** From a plain `node` at the repo root, with nothing
|
||||
linked, the built loader reads all 22 cubes under `cubes/` with zero errors. The
|
||||
`ERR_MODULE_NOT_FOUND` gotcha in `CLAUDE.md` is retired.
|
||||
|
||||
## Phase 5 — proof of concept: `packages/cubes-core` — **done**
|
||||
|
||||
Depends on Phase 4 shipping first — the bundle cannot declare
|
||||
`@bitsquare/nopy-cube` as a dependency until it exists, and the publish-lane fix
|
||||
@@ -478,19 +541,50 @@ has to be in place before either package is published.
|
||||
`nopy-cube` references from Phase 4 are separate.)
|
||||
8. Biome already lints `cubes/**/*.mjs` from the root; only the path changes.
|
||||
|
||||
### Verifying the PoC
|
||||
### What differed from the plan
|
||||
|
||||
- **In-workspace:** `pnpm --filter @bitsquare/nopy run nopy -P` from the repo
|
||||
root lists `net:tailscale`, `apt:install`, … and prints deploy commands whose
|
||||
`--chdir` points into `node_modules/@bitsquare/cubes-core/cubes/…`.
|
||||
- **Out-of-workspace (the real test):** `npm pack` the bundle, install the
|
||||
tarball into a throwaway directory with a `.nopyrc.json` naming it, install
|
||||
`nopy` *globally*, and run `nopy -P`. This is what actually exercises Phase 4 —
|
||||
a manifest resolving its import from a `node_modules` tree that has no
|
||||
`@bitsquare/nopy` in it. Check the installed tarball's `package.json` really
|
||||
carries a concrete `@bitsquare/nopy-cube` range and not `workspace:*`.
|
||||
- **Step 2's optional migration was done.** All 22 manifests now import
|
||||
`{ Manifest }` from `@bitsquare/nopy-cube`, not `{ cubes }` from
|
||||
`@bitsquare/nopy`. Optional for correctness, but it is the only version of the
|
||||
PoC that proves anything: leaving the old import in place would have resolved
|
||||
through the CLI that happens to sit in the same tree.
|
||||
- **`uniqid` had to move too.** Two manifests use it (`admin:hostname` bare,
|
||||
`user:add` via `cubes.uniqid`), so `src/cubes/utils.ts` and its test went to
|
||||
`nopy-cube` alongside `types.ts`, and `uniqid` joined the authoring barrel.
|
||||
Otherwise one migrated manifest would still have been importing the CLI.
|
||||
- **`files` needs a log exclusion.** Cubes that have been run leave a gitignored
|
||||
`pyinfra-debug.log` next to `deploy.py`; gitignore does not filter an npm
|
||||
tarball. `"files": ["cubes", "!cubes/**/*.log", …]` does. Verified: 22
|
||||
manifests, 22 deploy scripts, 0 logs in the packed artefact.
|
||||
- **`verify-pack.mjs` picks the bundle up for free** — it walks every non-private
|
||||
`packages/*`, so `cubes-core`'s `workspace:*` edge is checked like nopy's.
|
||||
|
||||
## Phase 6 — documentation
|
||||
### Verifying the PoC — done
|
||||
|
||||
- **In-workspace:** the built loader, run from the repo root against the new
|
||||
root `.nopyrc.json`, reads 22 cubes with 0 errors and reports
|
||||
`source: { type: 'package', packageName: '@bitsquare/cubes-core', dir:
|
||||
'…/node_modules/@bitsquare/cubes-core/cubes' }` — the pnpm symlink path, not a
|
||||
plain directory.
|
||||
- **Out-of-workspace (the real test):** `pnpm pack` for `nopy-cube`, `nopy` and
|
||||
`cubes-core`, then **`npm install`** of all three tarballs into a throwaway
|
||||
directory with a `.nopyrc.json` naming only the bundle. npm is the strict test
|
||||
here — it does not understand `workspace:`, so a leaked range fails the install
|
||||
outright. It installed clean, and the installed
|
||||
`@bitsquare/nopy/package.json` carries `"@bitsquare/nopy-cube":
|
||||
"1.0.0-alpha0"`. `nopy install -l session.json -P -D` then resolved
|
||||
`apt:essentials` and printed a `--chdir` into
|
||||
`node_modules/@bitsquare/cubes-core/cubes/apt/essentials`. Since the loader
|
||||
aborts on any manifest error and this run did not, all 22 manifests imported
|
||||
`@bitsquare/nopy-cube` and `zod` successfully from a tree containing no
|
||||
workspace links.
|
||||
|
||||
Note for anyone repeating this: `-P` on its own is interactive, and a replay
|
||||
still prompts for anything a manifest declares in `secrets` (they are never
|
||||
persisted to a session) — `net:tailscale` will sit there waiting. Use a
|
||||
session file with a cube that has no secrets, or answer the prompt.
|
||||
|
||||
## Phase 6 — documentation — **done**
|
||||
|
||||
- `CLAUDE.md`: the repo table gains two rows (`packages/nopy-cube`,
|
||||
`packages/cubes-core`) and loses the `cubes/` one; "The two packages do not
|
||||
@@ -503,6 +597,21 @@ has to be in place before either package is published.
|
||||
plus the ordering constraint — `nopy-cube` releases before anything that
|
||||
depends on it.
|
||||
|
||||
Beyond the list: `CLAUDE.md` also needed the `typecheck` command corrected
|
||||
(`tsc --build`, not `--noEmit` — see Phase 4), a note on the vitest source alias
|
||||
and the coverage exclusion, and three entries under *Known drift*. `README.PUBLISH.md`
|
||||
absorbed the whole publish-lane rework, not just the tag prefixes: `pnpm publish`
|
||||
over `npm publish` and why, the two-pass version stamping, `verify-pack.mjs`,
|
||||
`publish-order.mjs`, `linked-deps.mjs`, and a local rehearsal recipe that uses
|
||||
**npm** to install the tarballs precisely because npm is the one that rejects a
|
||||
leaked `workspace:` range.
|
||||
|
||||
One workflow change came out of writing this up: `ci.yml` now runs
|
||||
`verify-pack.mjs` too. It was only in the two publish workflows, which means a
|
||||
leaked range would have failed the release rather than the pull request that
|
||||
introduced it — the wrong end of the process for a mistake that is free to catch
|
||||
early.
|
||||
|
||||
## Testing
|
||||
|
||||
The coverage gate (85 % branches/functions, 80 % lines/statements, per package)
|
||||
|
||||
@@ -54,3 +54,29 @@ This document tracks the major refactoring of the `nopy` package.
|
||||
- `VariableAssignment` offers every schema key, not only the ones carrying a default, and shows the value the run would actually use as the initial.
|
||||
- **Proposed Solution**: (Done)
|
||||
|
||||
|
||||
### 6. Make variable assignment a first-class concept
|
||||
- **Status**: ✅ Completed
|
||||
- **Goal**: Give a variable an identity and a provenance, instead of inferring both from which bag it happened to sit in.
|
||||
- **Rationale**: Item 5 left precedence encoded as the field order of an object literal inside `Variables.get()` — `defaults`, then `global`, then `prompts`, then `params`. Nothing named the ranking, nothing could be asked where a value came from, and a replay had to be smuggled into the `prompts` bag because there was no origin that meant "recorded". Every question that followed — what should a session record, which values are safe to print — needed provenance to answer.
|
||||
- **Context**:
|
||||
- `Assignment { value, origin }` and an `Origin` ranked `default(0) < env(1) < session(2) < prompt(3) < param(4)`. Precedence is now data, not the order lines appear in.
|
||||
- `Variable` is a class over an assignment list. `assignments` is the true history, newest first and never reordered; `ordered` is a *stable* sort of it by origin rank, and `value`/`origin` read the head of that. Stability is what makes the two views coexist: same-origin ties keep the newest in front while the value it displaced stays visible.
|
||||
- The `global` bag is gone. Config `env` is seeded per cube as a real assignment at origin `env`, so `variables.get('global')` — a cube id that was never a cube — is no longer a thing.
|
||||
- Replay assigns at origin `session`, which outranks `env` and `default` on its own. The `prompts`-bag workaround is deleted.
|
||||
- A session records `Variables.persistable()` — every effective value, not just prompted ones. A `-D` run used to record nothing and replay by re-deriving from whatever the defaults said at replay time.
|
||||
- **Trade-off accepted**: recorded values now outrank the current `.nopyrc.json` `env` and the current schema defaults, so editing either no longer leaks into an existing session's replay. That is the point of a snapshot, but it does mean picking up a new default requires re-recording.
|
||||
- **Proposed Solution**: (Done)
|
||||
|
||||
### 7. Manifest-declared secrets
|
||||
- **Status**: ✅ Completed
|
||||
- **Goal**: Let a manifest say which schema keys hold sensitive values, and act on it.
|
||||
- **Rationale**: Item 6 made sessions record everything, which forced the question of what must *not* be recorded. The codebase already had an answer of sorts — `outputExecutionPlan` masked any variable whose name contained "password" — that missed `TOKEN`, `PSK` and `AUTH_KEY`, and was defeated anyway by the unmasked command printed one line above it.
|
||||
- **Context**:
|
||||
- `Manifest.secrets?: string[]`, validated at load: an entry that is not a key of `schema` is a manifest error and aborts the run, so a typo cannot silently leave a value unprotected.
|
||||
- Deliberately a plain array, not zod metadata. `.meta()` and `.describe()` store into `z.globalRegistry`, which is per-copy — a manifest built by a different zod copy would look up empty. Fail-open is fine for a missing prompt label and unacceptable for a secret marker.
|
||||
- `maskCommand()` replaces declared `--data` values and the SSH `--password` in the command string itself, and is wired into `--print-only`, the dry-run plan and the debug log. The `nopy` logger runs at `lowestLevel: 'debug'`, so that last one was printing credentials on every run.
|
||||
- Secrets are excluded from `persistable()`, so a replay has a gap where one used to be. `fillSessionGaps` prompts for `requiredKeys() ∪ secrets`; under `-D` it fails naming them, consistent with item 5's fail-fast.
|
||||
- **Scope limit**: `secrets` keeps a value out of what nopy writes. The value is still on pyinfra's command line (visible in `ps`), still echoed by the variable form, and a `.default()` is still plain text in the manifest. Documented rather than fixed — the first is inherent to pyinfra's interface.
|
||||
- **Bug fixed along the way**: `cubes/user/add` generated a random password as its schema `.default()`. Because the key had a default it was never in `requiredKeys()`, and because a generated default is re-evaluated on every read, an unattended run created an account with a credential nobody had seen and a replay created a different one again. It is now the literal `changeme`.
|
||||
- **Proposed Solution**: (Done)
|
||||
|
||||
@@ -294,8 +294,9 @@ export default {
|
||||
2. **Document your cubes** - Add comments explaining what each cube does
|
||||
3. **Use environment variables** - Make sessions reusable across environments
|
||||
4. **Extract common config** - Share configuration across multiple sessions
|
||||
5. **Version control** - Both formats work well with git
|
||||
5. **Version control** - Both formats work well with git, but a recorded session holds every value its run used; read one before committing it
|
||||
6. **Validate at runtime** - The loader validates the structure regardless of format
|
||||
7. **Leave secrets out** - Declare them in the manifest instead, and let the replay ask
|
||||
|
||||
## Session Schema
|
||||
|
||||
@@ -321,3 +322,16 @@ interface AuthSession {
|
||||
username?: string;
|
||||
}
|
||||
```
|
||||
|
||||
A session nopy *writes* holds, per cube, every value that cube ran with — what
|
||||
was typed, what came from `.nopyrc.json`, what a dependency supplied, and what
|
||||
fell through to the schema's `.default()`. Two things are deliberately absent and
|
||||
are asked for again on replay: the SSH password, and any key the cube's manifest
|
||||
listed under `secrets`.
|
||||
|
||||
A session you write by hand is under no such obligation — `variables` may hold as
|
||||
few keys as you like, and anything missing resolves the usual way. Note that a
|
||||
key declared a secret is prompted for whether or not the session carries a value:
|
||||
writing one in only pre-fills the prompt, it does not skip it. The variable form
|
||||
shows what it is editing, so a secret you put in a session file appears on screen
|
||||
as well as on disk.
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rm -rf dist .tsbuildinfo",
|
||||
"build": "tsc",
|
||||
"build": "tsc && cp src/cubes/*.mjs dist/cubes/",
|
||||
"prepack": "pnpm run build",
|
||||
"link:local": "pnpm run build && npm link",
|
||||
"nopy": "tsx src/nopy.cli.ts",
|
||||
@@ -54,6 +54,7 @@
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bitsquare/nopy-cube": "workspace:*",
|
||||
"@logtape/logtape": "^2.2.4",
|
||||
"commander": "^15.0.0",
|
||||
"enquirer": "^2.4.1",
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
* @module cubes/dependencies
|
||||
*/
|
||||
|
||||
import type { Cube, CubeVariables, HookContext } from '@bitsquare/nopy-cube';
|
||||
import { getLogger } from '@logtape/logtape';
|
||||
import type { Variables } from '../nopy.common.js';
|
||||
import type { NopyConfig } from '../nopy.config.js';
|
||||
import type { DeployCall } from '../nopy.executor.js';
|
||||
import { VariableAssignment } from '../nopy.prompts.js';
|
||||
import type { CubeSession, NopySession } from '../nopy.session.js';
|
||||
import type { Cube, CubeVariables, HookContext } from './types.js';
|
||||
|
||||
const log = getLogger(['nopy', 'resolution']);
|
||||
|
||||
@@ -38,6 +38,12 @@ export class BuildContext {
|
||||
} = {}
|
||||
) {}
|
||||
|
||||
/** Required schema keys that nothing has supplied a value for. */
|
||||
private missingRequired(cube: Cube): string[] {
|
||||
const resolved = this.variables.get(cube.id);
|
||||
return cube.requiredKeys().filter((key) => resolved[key] === undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails a non-interactive run that cannot fill a required variable.
|
||||
*
|
||||
@@ -45,8 +51,7 @@ export class BuildContext {
|
||||
* `--data`, and the deploy script would read `None` off `host.data`.
|
||||
*/
|
||||
private assertVariablesComplete(cube: Cube): void {
|
||||
const resolved = this.variables.get(cube.id);
|
||||
const missing = cube.requiredKeys().filter((key) => resolved[key] === undefined);
|
||||
const missing = this.missingRequired(cube);
|
||||
if (missing.length === 0) return;
|
||||
|
||||
const [one, them] =
|
||||
@@ -58,6 +63,43 @@ export class BuildContext {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks for the variables a replay cannot supply on its own.
|
||||
*
|
||||
* Two kinds. Required keys can be absent because the session predates them or
|
||||
* was recorded by a `--use-defaults` run. Secrets are absent by design: they
|
||||
* are never written to a session, so replaying without asking would deploy a
|
||||
* cube with the key missing — or, for a secret carrying a default, with a
|
||||
* value silently different from the run being replayed.
|
||||
*
|
||||
* Secrets are asked for even when a default did fill them in, which is why
|
||||
* this cannot key off "has no value": the whole point is that the recorded
|
||||
* answer is gone and only the user knows what it was.
|
||||
*/
|
||||
private async fillSessionGaps(cube: Cube): Promise<void> {
|
||||
const gaps = [...new Set([...this.missingRequired(cube), ...cube.secrets])];
|
||||
if (gaps.length === 0) return;
|
||||
|
||||
if (this.options.useDefaults) {
|
||||
throw new Error(
|
||||
`Cube "${cube.id}" cannot be replayed with --use-defaults: ${gaps.join(', ')} ` +
|
||||
'would have to be entered. Secrets are never recorded in a session. ' +
|
||||
'Replay without --use-defaults, or set the values under "env" in .nopyrc.json.'
|
||||
);
|
||||
}
|
||||
|
||||
log.debug('Filling session gaps', { cubeId: cube.id, gaps });
|
||||
await VariableAssignment(cube, this.variables, { keys: gaps });
|
||||
|
||||
// A cancelled form leaves the run short of a value it cannot invent.
|
||||
const stillMissing = this.missingRequired(cube);
|
||||
if (stillMissing.length > 0) {
|
||||
throw new Error(
|
||||
`Cube "${cube.id}" is missing ${stillMissing.join(', ')} and cannot be deployed.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a cube, its dependencies, and hooks recursively
|
||||
*/
|
||||
@@ -73,20 +115,22 @@ export class BuildContext {
|
||||
|
||||
log.debug('Resolving cube', { cubeId, host });
|
||||
|
||||
// 1. Assign overrides and defaults
|
||||
// 1. Declare secrets, then assign overrides and defaults. Declaring first
|
||||
// means even the config `env` seeded on the cube's first assignment is
|
||||
// already marked, so nothing reaches a session or a log unredacted.
|
||||
this.variables.declareSecrets(cubeId, cube.secrets);
|
||||
if (Object.keys(overrides).length > 0) {
|
||||
this.variables.assign(cubeId, 'params', overrides);
|
||||
this.variables.assign(cubeId, 'param', overrides);
|
||||
}
|
||||
this.variables.assign(cubeId, 'defaults', cube.getDefaults());
|
||||
this.variables.assign(cubeId, 'default', cube.getDefaults());
|
||||
|
||||
// 2. Variable collection
|
||||
if (this.options.isSessionReplay) {
|
||||
// Recorded answers go back into the scope they came from, so a replay
|
||||
// reproduces them even when `env` sets the same key to something else.
|
||||
const sessionCube = this.session.cubes.find((c) => c.key === cubeId);
|
||||
if (sessionCube) {
|
||||
this.variables.assign(cubeId, 'prompts', sessionCube.variables);
|
||||
this.variables.assign(cubeId, 'session', sessionCube.variables);
|
||||
}
|
||||
await this.fillSessionGaps(cube);
|
||||
} else if (this.options.useDefaults) {
|
||||
log.debug('Skipping prompts, using defaults', { cubeId });
|
||||
this.assertVariablesComplete(cube);
|
||||
@@ -155,13 +199,18 @@ export class BuildContext {
|
||||
cwd: cube.dir,
|
||||
command,
|
||||
env: cubeVars,
|
||||
secrets: cube.secrets,
|
||||
dependencies: [],
|
||||
});
|
||||
|
||||
if (!this.cubeSessions.some((s) => s.key === cubeId)) {
|
||||
// Every value the run settled on, not just the prompted ones — otherwise a
|
||||
// `--use-defaults` run records nothing and replaying it re-derives from
|
||||
// whatever the defaults and `env` happen to say now. Secrets are the one
|
||||
// exclusion; a replay asks for those again.
|
||||
this.cubeSessions.push({
|
||||
key: cubeId,
|
||||
variables: this.variables.get(cubeId, 'prompts'),
|
||||
variables: this.variables.persistable(cubeId),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user