diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 0c461f3..e246ba2 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -81,6 +81,13 @@ jobs: echo "::endgroup::" done + - name: Verify the packed manifests + # `workspace:*` is mandatory in the manifests but meaningless to npm, so + # a range that survives into a tarball is an install failure for every + # consumer. The publish workflows run this too; running it here is what + # puts the failure on the pull request instead of on the release. + run: node scripts/verify-pack.mjs + - name: Upload coverage reports if: always() continue-on-error: true diff --git a/.gitea/workflows/publish-snapshot.yml b/.gitea/workflows/publish-snapshot.yml index ca03453..234f503 100644 --- a/.gitea/workflows/publish-snapshot.yml +++ b/.gitea/workflows/publish-snapshot.yml @@ -1,5 +1,5 @@ -# Every commit that lands on `main` publishes a prerelease of both packages to -# the Gitea npm registry under the `main` dist-tag: +# Every commit that lands on `main` publishes a prerelease of every publishable +# package to the Gitea npm registry under the `main` dist-tag: # # pnpm add @bitsquare/nopy@main # @@ -78,6 +78,11 @@ jobs: # Explicit, so the publish step can skip lifecycle scripts entirely. run: pnpm run build + - name: Verify the packed manifests + # Packages link to each other with `workspace:*`, which npm cannot + # install. Proves on the tarball that pack rewrote it. + run: node scripts/verify-pack.mjs + - name: Authenticate against the Gitea registry run: | set -euo pipefail @@ -97,23 +102,35 @@ jobs: export npm_config_userconfig="$NPMRC" : "${GITHUB_STEP_SUMMARY:=/dev/null}" short_sha=$(git rev-parse --short=7 HEAD) + # Dependencies first, so the registry never briefly holds a package + # whose dependency has not landed yet. + dirs=$(node scripts/publish-order.mjs) - for dir in packages/*/; do - name=$(node -p "require('./${dir}package.json').name") - base=$(node -p "require('./${dir}package.json').version") + # Pass 1: stamp every manifest before anything is packed. `pnpm + # publish` substitutes `workspace:*` with the version the linked + # package declares at pack time, so nopy-cube has to be carrying its + # snapshot version by the time nopy is packed. + for dir in $dirs; do + base=$(node -p "require('./${dir}/package.json').version") # `g` prefix keeps the identifier a valid semver one even when the # abbreviated sha happens to be all digits. version="${base}-main.${{ github.run_number }}.g${short_sha}" + (cd "$dir" && npm pkg set "version=${version}") + done + + # Pass 2: publish. + for dir in $dirs; do + name=$(node -p "require('./${dir}/package.json').name") + version=$(node -p "require('./${dir}/package.json').version") echo "::group::${name}@${version}" if npm view "${name}@${version}" version --registry "$REGISTRY" >/dev/null 2>&1; then echo "Already published — skipping (this is a re-run of the same workflow)." else - ( - cd "$dir" - npm pkg set "version=${version}" - npm publish --ignore-scripts --tag main --registry "$REGISTRY" - ) + # pnpm, not npm: npm ships `workspace:*` verbatim and the install + # then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because + # stamping the versions above left the tree dirty. + (cd "$dir" && pnpm publish --ignore-scripts --no-git-checks --tag main --registry "$REGISTRY") fi echo "::endgroup::" diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index c056cef..609b003 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,10 +1,15 @@ # Tag-driven release of a single package. # # git tag nopy-v1.2.0 && git push origin nopy-v1.2.0 +# git tag nopy-cube-v1.2.0 && git push origin nopy-cube-v1.2.0 # git tag keyman-v1.2.0 && git push origin keyman-v1.2.0 # # The tag is the source of truth for *which* package ships; package.json is the # source of truth for the version, and the two must agree or the run fails. +# +# Packages that link to each other release dependency-first — `nopy-cube` before +# `nopy` — because the linked version is resolved at pack time. The run refuses +# to publish otherwise. # A version with a prerelease part (1.2.0-rc.1) publishes under `next` instead # of `latest`. # @@ -108,6 +113,31 @@ jobs: - name: Install run: pnpm install --frozen-lockfile + - name: Check the linked workspace packages are already released + env: + NAME: ${{ steps.target.outputs.name }} + DIR: ${{ steps.target.outputs.dir }} + run: | + set -euo pipefail + # `pnpm publish` turns `workspace:*` into the version the linked + # package declares at this commit. If that version is not on the + # registry yet, the release installs to a broken tree — and npmjs + # only lets you unpublish for 72 hours. Release the dependency first: + # nopy-cube, then nopy, then any bundle. + # + # npmjs only: it is the irreversible one, and it needs no credentials + # to read, which this step does not have yet. + missing=0 + for spec in $(node scripts/linked-deps.mjs "$DIR" | tr ' ' '@'); do + if npm view "$spec" version --registry "$NPMJS_REGISTRY" >/dev/null 2>&1; then + echo "${spec} is published" + else + echo "::error::${NAME} depends on ${spec}, which is not on npmjs. Release it first." + missing=1 + fi + done + exit "$missing" + - name: Lint run: pnpm run lint:ci @@ -121,6 +151,11 @@ jobs: # Explicit, so the publish steps can skip lifecycle scripts entirely. run: pnpm run build + - name: Verify the packed manifests + # Packages link to each other with `workspace:*`, which npm cannot + # install. Proves on the tarball that pack rewrote it. + run: node scripts/verify-pack.mjs + - name: Publish to the Gitea registry env: NAME: ${{ steps.target.outputs.name }} @@ -139,7 +174,10 @@ jobs: if npm view "${NAME}@${VERSION}" version --registry "$GITEA_REGISTRY" >/dev/null 2>&1; then echo "${NAME}@${VERSION} is already on Gitea — skipping." else - (cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --registry "$GITEA_REGISTRY") + # pnpm, not npm: npm ships `workspace:*` verbatim and the install + # then fails with EUNSUPPORTEDPROTOCOL. --no-git-checks because a + # tag build is a detached HEAD. + (cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --registry "$GITEA_REGISTRY") fi - name: Publish to npmjs @@ -162,7 +200,7 @@ jobs: else # No --provenance: that needs GitHub Actions OIDC, which Gitea has no # equivalent for. - (cd "$DIR" && npm publish --ignore-scripts --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY") + (cd "$DIR" && pnpm publish --ignore-scripts --no-git-checks --tag "$DIST_TAG" --access public --registry "$NPMJS_REGISTRY") fi - name: Remove the registry credentials diff --git a/.nopyrc.json b/.nopyrc.json index f51048b..a015fff 100644 --- a/.nopyrc.json +++ b/.nopyrc.json @@ -1,6 +1,7 @@ { "hosts": [], - "cubeDirs": ["./cubes"], + "cubeDirs": [], + "cubePackages": ["@bitsquare/cubes-core"], "env": {}, "log": { "verbosity": "info", diff --git a/CLAUDE.md b/CLAUDE.md index 33deed2..2ac56a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,26 +4,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this repo is -A pnpm workspace holding two independently published CLIs plus the pyinfra -deployment units one of them runs: +A pnpm workspace holding two independently published CLIs, the authoring package +their deployment units are written against, and one bundle of those units: -| Path | Package | Binary | Role | -| ----------------- | ------------------- | -------- | -------------------------------------------------------- | -| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution | -| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` | -| `cubes/` | — | — | the deployment units `nopy` runs (not published) | +| Path | Package | Binary | Role | +| --------------------- | ---------------------- | -------- | -------------------------------------------------------- | +| `packages/nopy` | `@bitsquare/nopy` | `nopy` | interactive pyinfra script management and execution | +| `packages/keyman` | `@bitsquare/keyman` | `keyman` | SSH key management, shelling out to `age` / `ssh-keygen` | +| `packages/nopy-cube` | `@bitsquare/nopy-cube` | — | the authoring surface a `manifest.mjs` imports | +| `packages/cubes-core` | `@bitsquare/cubes-core`| — | the core cube bundle (22 cubes), no TypeScript | -The root package is private; only `packages/*` ship. The two packages do not -depend on each other. +The root package is private; everything under `packages/` ships. `keyman` stands +alone, but `nopy` and `cubes-core` both depend on `nopy-cube` (`workspace:*`), so +publish order matters — see *Releasing*. + +`cubes-core` is consumed the way a third party would consume it: the root +`.nopyrc.json` names it in `cubePackages`, and the loader reads it out of +`node_modules`. There is no `cubes/` directory at the repo root any more. ## Commands ```sh pnpm install # also installs the git hooks via simple-git-hooks -pnpm run build # tsc --build across both packages (project references) -pnpm run typecheck # tsc --build --noEmit +pnpm run build # tsc --build across the TS packages (project references) +pnpm run typecheck # tsc --build (see below — it really does emit) pnpm run lint # biome check . (lint:fix / lint:ci variants) -pnpm test # vitest run, both packages +pnpm test # vitest run, every package with tests pnpm run test:coverage # vitest with the coverage gate pnpm run coverage:summary # renders the last coverage run as a Markdown table ``` @@ -41,6 +47,12 @@ pnpm --filter @bitsquare/keyman run keyman `typescript` is the 7.x native compiler, so `tsc` *is* the fast one — there is no separate `tsgo` binary. +`typecheck` is plain `tsc --build`, not `--noEmit`. Once a project has +`references`, `--noEmit` is rejected outright (TS6310: *referenced project may +not disable emit*) — a composite project has to emit the declarations its +dependents read. So the typecheck writes `dist` as a side effect; it is +gitignored, and the upside is that the gate now also proves the build works. + ## Verification gate `lint:ci` → `typecheck` → `test:coverage` is one gate, run in three places: the @@ -55,7 +67,14 @@ locally and on the runner. Barrel files (`src/index.ts`, `src/cubes/index.ts`, `src/nopy.cubes.ts`) and the Commander argv wiring (`src/*.cli.ts`) are excluded; adding logic to those files means moving it somewhere covered. -Both packages set `pool: 'forks'` because tests use `process.chdir()` — most +nopy's vitest config aliases `@bitsquare/nopy-cube` to that package's **source**, +not to the workspace link (which points at a `dist` that only exists after a +build), so the gate does not depend on build ordering and can never run against a +stale artefact. The same config excludes `**/nopy-cube/**` from coverage — without +it nopy's numbers absorb another package's files. `cubes-core` has no tests of its +own; the loader tests in nopy cover the contract it implements. + +The three TS packages set `pool: 'forks'` because tests use `process.chdir()` — most loader/config tests build a throwaway tree under `os.tmpdir()` and chdir into it, since discovery is driven entirely by the working directory. @@ -68,22 +87,43 @@ One pass per invocation, `nopy.main.ts` orchestrating: Per-property strategy comes from the child's `resolution` block (`merge` is the default: arrays concatenate and dedupe, objects deep-merge; `override` replaces). Only properties listed in `PATH_PROPERTIES` (`cubeDirs`) get - relative paths resolved against their own config file's directory. **Throws** + relative paths resolved against their own config file's directory; + `cubePackages` needs the same origin for a different reason, so each entry is + normalised into a `CubePackageRef {spec, from}` — `from` is the directory of + the config that named it, which is where the package gets resolved from. + **Throws** if no config file exists anywhere — which is why `nopy.cli.ts` calls it lazily inside the action, so `--help`/`--version` work outside a project. -2. **`cubes/loader.ts`** — `findCubeDirectories()` unions `config.cubeDirs` with - every ancestor directory holding a `.npcubes` marker, then scans each - recursively (skipping dotted dirs and `node_modules`). A directory is a cube +2. **`cubes/packages.ts`** — `resolveCubePackages()` turns each `CubePackageRef` + into a package root plus the directories its `nopy.cubes` field declares. + Resolution goes through `createRequire(...).resolve.paths()` + `existsSync`, + deliberately bypassing the `exports` map: a bundle ships directories and has + no entry point to declare. `existsSync` also follows the symlink pnpm plants + at `node_modules/`, which a `readdir` scan skips outright (it reports + `isSymbolicLink()`, not `isDirectory()`). A missing package, an unreadable + manifest, a missing `nopy.cubes`, a directory that does not exist, and an + entry pointing outside the package root are all errors, never silent skips. + Duplicate refs are deduped here, last-wins, because `mergeValue` only dedupes + arrays of primitives and these are objects. +3. **`cubes/loader.ts`** — `findCubeRoots()` unions `config.cubeDirs`, the + directories from `cubePackages`, and every ancestor directory holding a + `.npcubes` marker, then scans each recursively (skipping dotted dirs and + `node_modules`). A directory is a cube when it holds both a manifest (`manifest.mjs` or `*.manifest.mjs`) and a deploy script (`deploy.py` or `*.deploy.py`); manifests are loaded by dynamic `import()`. Cube id = `manifest.id` → a `[id]` prefix in `manifest.name` → the directory basename. Ids are flat and need not mirror the path - (`cubes/network/tailscale` declares `net:tailscale`). Duplicate ids and bad - manifests become entries in `errors`, which aborts the run. -3. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and + (`cubes/network/tailscale` declares `net:tailscale`), and they are claimed + **globally**, not per source: a duplicate is a hard error naming every + claimant, with no precedence rule and no shadowing. Each cube carries a + `source` — `{type: 'dir', dir}` or `{type: 'package', packageName, dir}` — + which is what makes that error legible when the collision is between a local + tree and an installed bundle. Duplicate ids and bad manifests become entries + in `errors`, which aborts the run. +4. **`nopy.workflow.ts`** — picks interactive, file-replay, or history-replay and normalises all three into a `WorkflowResult`. Replays never re-prompt except for passwords (never persisted) and a missing host. -4. **`cubes/dependencies.ts` → `BuildContext.resolveCube()`** — the core. +5. **`cubes/dependencies.ts` → `BuildContext.resolveCube()`** — the core. Recursive, per (cube, host): assign params and schema defaults → collect variables (prompt, or read them back from the session on replay) → run `before` hooks → resolve `manifest.dependencies(vars)` (dynamic: it receives @@ -92,33 +132,71 @@ One pass per invocation, `nopy.main.ts` orchestrating: `${cubeId}:${host}` set makes emission idempotent. Hooks get a `HookContext` whose `exec(id, vars)` re-enters `resolveCube`, so a hook can pull in a cube that is not a declared dependency. -5. **`nopy.executor.ts`** — runs the built `pyinfra -y --data K=V ... --chdir