[refactor] moving cubes into own package"
Publish snapshot / snapshot (push) Successful in 1m2s

[fix] default parameter run records parameters in session for replay[fix] remove default parameters for several cubes
This commit is contained in:
Benjamin Diedrichsen
2026-07-28 12:18:10 +02:00
parent ac050c4459
commit 6ecb2c366f
130 changed files with 3386 additions and 520 deletions
+59 -10
View File
@@ -3,13 +3,13 @@
* @module cubes/dependencies
*/
import type { Cube, CubeVariables, HookContext } from '@bitsquare/nopy-cube';
import { getLogger } from '@logtape/logtape';
import type { Variables } from '../nopy.common.js';
import type { NopyConfig } from '../nopy.config.js';
import type { DeployCall } from '../nopy.executor.js';
import { VariableAssignment } from '../nopy.prompts.js';
import type { CubeSession, NopySession } from '../nopy.session.js';
import type { Cube, CubeVariables, HookContext } from './types.js';
const log = getLogger(['nopy', 'resolution']);
@@ -38,6 +38,12 @@ export class BuildContext {
} = {}
) {}
/** Required schema keys that nothing has supplied a value for. */
private missingRequired(cube: Cube): string[] {
const resolved = this.variables.get(cube.id);
return cube.requiredKeys().filter((key) => resolved[key] === undefined);
}
/**
* Fails a non-interactive run that cannot fill a required variable.
*
@@ -45,8 +51,7 @@ export class BuildContext {
* `--data`, and the deploy script would read `None` off `host.data`.
*/
private assertVariablesComplete(cube: Cube): void {
const resolved = this.variables.get(cube.id);
const missing = cube.requiredKeys().filter((key) => resolved[key] === undefined);
const missing = this.missingRequired(cube);
if (missing.length === 0) return;
const [one, them] =
@@ -58,6 +63,43 @@ export class BuildContext {
);
}
/**
* Asks for the variables a replay cannot supply on its own.
*
* Two kinds. Required keys can be absent because the session predates them or
* was recorded by a `--use-defaults` run. Secrets are absent by design: they
* are never written to a session, so replaying without asking would deploy a
* cube with the key missing — or, for a secret carrying a default, with a
* value silently different from the run being replayed.
*
* Secrets are asked for even when a default did fill them in, which is why
* this cannot key off "has no value": the whole point is that the recorded
* answer is gone and only the user knows what it was.
*/
private async fillSessionGaps(cube: Cube): Promise<void> {
const gaps = [...new Set([...this.missingRequired(cube), ...cube.secrets])];
if (gaps.length === 0) return;
if (this.options.useDefaults) {
throw new Error(
`Cube "${cube.id}" cannot be replayed with --use-defaults: ${gaps.join(', ')} ` +
'would have to be entered. Secrets are never recorded in a session. ' +
'Replay without --use-defaults, or set the values under "env" in .nopyrc.json.'
);
}
log.debug('Filling session gaps', { cubeId: cube.id, gaps });
await VariableAssignment(cube, this.variables, { keys: gaps });
// A cancelled form leaves the run short of a value it cannot invent.
const stillMissing = this.missingRequired(cube);
if (stillMissing.length > 0) {
throw new Error(
`Cube "${cube.id}" is missing ${stillMissing.join(', ')} and cannot be deployed.`
);
}
}
/**
* Resolves a cube, its dependencies, and hooks recursively
*/
@@ -73,20 +115,22 @@ export class BuildContext {
log.debug('Resolving cube', { cubeId, host });
// 1. Assign overrides and defaults
// 1. Declare secrets, then assign overrides and defaults. Declaring first
// means even the config `env` seeded on the cube's first assignment is
// already marked, so nothing reaches a session or a log unredacted.
this.variables.declareSecrets(cubeId, cube.secrets);
if (Object.keys(overrides).length > 0) {
this.variables.assign(cubeId, 'params', overrides);
this.variables.assign(cubeId, 'param', overrides);
}
this.variables.assign(cubeId, 'defaults', cube.getDefaults());
this.variables.assign(cubeId, 'default', cube.getDefaults());
// 2. Variable collection
if (this.options.isSessionReplay) {
// Recorded answers go back into the scope they came from, so a replay
// reproduces them even when `env` sets the same key to something else.
const sessionCube = this.session.cubes.find((c) => c.key === cubeId);
if (sessionCube) {
this.variables.assign(cubeId, 'prompts', sessionCube.variables);
this.variables.assign(cubeId, 'session', sessionCube.variables);
}
await this.fillSessionGaps(cube);
} else if (this.options.useDefaults) {
log.debug('Skipping prompts, using defaults', { cubeId });
this.assertVariablesComplete(cube);
@@ -155,13 +199,18 @@ export class BuildContext {
cwd: cube.dir,
command,
env: cubeVars,
secrets: cube.secrets,
dependencies: [],
});
if (!this.cubeSessions.some((s) => s.key === cubeId)) {
// Every value the run settled on, not just the prompted ones — otherwise a
// `--use-defaults` run records nothing and replaying it re-derives from
// whatever the defaults and `env` happen to say now. Secrets are the one
// exclusion; a replay asks for those again.
this.cubeSessions.push({
key: cubeId,
variables: this.variables.get(cubeId, 'prompts'),
variables: this.variables.persistable(cubeId),
});
}
-30
View File
@@ -1,30 +0,0 @@
/**
* Factory functions for creating cube configurations
* @module cubes/factories
*/
import { type AnyObjectSchema, Manifest } from './types.js';
/**
* Creates a manifest configuration for a cube
*
* @param opts - Manifest options including name, schema, dependencies, and hooks
* @returns Manifest configuration object
*/
export function createManifest<Schema extends AnyObjectSchema>(
opts: Pick<Manifest<Schema>, 'name'> & Partial<Omit<Manifest<Schema>, 'name'>>
): Manifest<Schema> {
return Manifest(opts);
}
/**
* Alias for createManifest - for backwards compatibility with existing manifests
*/
export const manifest = createManifest;
/**
* @deprecated Use createManifest or manifest instead
*/
export const ManifestFactory = createManifest;
export { Manifest } from './types.js';
+22 -19
View File
@@ -6,34 +6,37 @@
* @module cubes
*/
// Dependencies
export { BuildContext } from './dependencies.js';
// Factory functions
export {
createManifest,
manifest,
} from './factories.js';
// Loader
export {
findCubeDirectories,
getCube,
loadCubes,
} from './loader.js';
// The authoring surface lives in its own package so that a cube bundle can
// depend on it without pulling the CLI in. Re-exported here so that
// `import { cubes } from '@bitsquare/nopy'` in a manifest keeps working.
export type {
AnyObjectSchema,
CubeSource,
CubeVariables,
DependencySpec,
Hook,
HookContext,
LoadResult,
} from './types.js';
// Types
} from '@bitsquare/nopy-cube';
export {
Cube,
createManifest,
Manifest,
manifest,
uniqid,
zodInner,
zodKind,
} from './types.js';
// Utilities
export { uniqid } from './utils.js';
} from '@bitsquare/nopy-cube';
// Dependencies
export { BuildContext } from './dependencies.js';
// Loader
export type { CubeRoot } from './loader.js';
export {
findCubeDirectories,
findCubeRoots,
getCube,
loadCubes,
} from './loader.js';
// Packages
export type { CubePackage } from './packages.js';
export { resolveCubePackages } from './packages.js';
+102 -17
View File
@@ -3,21 +3,55 @@
* @module cubes/loader
*/
import module from 'node:module';
import path from 'node:path';
import { Cube, type CubeSource, type LoadResult, type Manifest } from '@bitsquare/nopy-cube';
import { z } from 'zod';
import { fs } from 'zx';
import { loadConfig } from '../nopy.config.js';
import { Cube, type LoadResult, type Manifest } from './types.js';
import { resolveCubePackages } from './packages.js';
let hookRegistered = false;
/**
* Traverses upwards from the current working directory to the root
* and collects all directories that contain a `.npcubes` marker file.
* Installs the fallback resolver that lets a manifest in a bare directory
* import `@bitsquare/nopy-cube` or `zod` — see `resolve-hook.mjs`.
*
* Also includes directories specified in the `.nopyrc.json` configuration.
*
* @returns Array of absolute paths to directories containing cubes
* `module.register()` is process-global and cannot be undone, so this runs once
* and only when cubes are about to be imported. Registration failing is not
* worth aborting a run over: without the hook, a cube that needed it fails on
* its own import with a message that names the file.
*/
export function findCubeDirectories(): string[] {
function registerResolveHook(): void {
if (hookRegistered) return;
hookRegistered = true;
try {
// `from` is a URL inside this package, so the hook thread resolves the
// fallbacks out of the running CLI's own dependencies.
module.register('./resolve-hook.mjs', import.meta.url, { data: { from: import.meta.url } });
} catch {
// Nothing to do: the hook is a convenience, never load-bearing.
}
}
/** A directory to scan, and what put it in the list. */
export interface CubeRoot {
dir: string;
source: CubeSource;
}
/**
* Collects every root to scan for cubes:
*
* - `cubeDirs` from the merged configuration,
* - every ancestor of the working directory holding a `.npcubes` marker file,
* - the cube directories of every package named in `cubePackages`.
*
* Only the last of those can fail — a missing directory is ignored, a missing
* package is not (see `resolveCubePackages`).
*/
export function findCubeRoots(): { roots: CubeRoot[]; errors: string[] } {
let currentDir = process.cwd();
const config = loadConfig();
const dirSet = new Set<string>(config.cubeDirs.map((dir) => path.resolve(process.cwd(), dir)));
@@ -38,7 +72,29 @@ export function findCubeDirectories(): string[] {
currentDir = parentDir;
}
return [...dirSet];
const roots: CubeRoot[] = [...dirSet].map((dir) => ({ dir, source: { type: 'dir', dir } }));
const { packages, errors } = resolveCubePackages(config.cubePackages);
for (const pkg of packages) {
for (const dir of pkg.dirs) {
roots.push({ dir, source: { type: 'package', packageName: pkg.name, dir } });
}
}
return { roots, errors };
}
/**
* The directories {@link findCubeRoots} would scan.
*
* Kept for callers that only want the paths; anything that needs to attribute
* a cube to where it came from should use `findCubeRoots` instead, which also
* reports the errors this one drops.
*
* @returns Array of absolute paths to directories containing cubes
*/
export function findCubeDirectories(): string[] {
return findCubeRoots().roots.map((root) => root.dir);
}
/**
@@ -56,10 +112,12 @@ interface CubeCandidate {
manifest: Manifest;
dir: string;
deployScript: string;
source: CubeSource;
}
/** What one root directory contributed. */
interface ScanResult {
root: CubeRoot;
candidates: CubeCandidate[];
errors: string[];
}
@@ -99,11 +157,23 @@ async function scanDirectory(currentDir: string, result: ScanResult): Promise<vo
manifest.id = cubeId;
manifest.schema = manifest.schema ?? z.object({});
// A `secrets` entry naming a key that is not in the schema protects
// nothing, and a typo in one is invisible at runtime — the value would
// just be persisted. Cheaper to refuse the cube than to ship the leak.
const unknown = (manifest.secrets ?? []).filter((key) => !(key in manifest.schema.shape));
if (unknown.length > 0) {
result.errors.push(
`Invalid manifest in ${manifestPath}: 'secrets' names ${unknown.join(', ')}, ` +
`which ${unknown.length === 1 ? 'is' : 'are'} not in the schema`
);
}
result.candidates.push({
id: cubeId,
manifest,
dir: currentDir,
deployScript: deployFile.name,
source: result.root.source,
});
}
} catch (err) {
@@ -118,9 +188,23 @@ async function scanDirectory(currentDir: string, result: ScanResult): Promise<vo
}
}
/** The message a duplicate id produces. Aborts the run — see `nopy.main.ts`. */
/**
* The message a duplicate id produces. Aborts the run — see `nopy.main.ts`.
*
* There is deliberately no precedence rule to fall back on: two cubes claiming
* one id are mutually exclusive, and the fix belongs upstream. So the message
* has to carry everything needed to go and make it, which means naming every
* claimant and how each got into the run.
*/
function duplicateError(id: string, group: CubeCandidate[]): string {
const where = group.map((c) => ` ${c.dir}`).join('\n');
const label = (candidate: CubeCandidate) =>
candidate.source.type === 'package' ? `package ${candidate.source.packageName}` : 'directory';
const width = Math.max(...group.map((candidate) => label(candidate).length));
const where = group
.map((candidate) => ` ${label(candidate).padEnd(width)} ${candidate.dir}`)
.join('\n');
return (
`Duplicate cube id '${id}' from ${group.length} sources:\n${where}\n` +
`Rename one of them, or remove a source from .nopyrc.json.`
@@ -137,20 +221,21 @@ function duplicateError(id: string, group: CubeCandidate[]): string {
* run, which is what makes the hard error testable.
*/
export async function loadCubes(): Promise<LoadResult> {
const cubesFolders = findCubeDirectories();
const { roots, errors: rootErrors } = findCubeRoots();
registerResolveHook();
const scans = await Promise.all(
cubesFolders.map(async (folder) => {
const result: ScanResult = { candidates: [], errors: [] };
if (fs.existsSync(folder)) {
await scanDirectory(folder, result);
roots.map(async (root) => {
const result: ScanResult = { root, candidates: [], errors: [] };
if (fs.existsSync(root.dir)) {
await scanDirectory(root.dir, result);
}
return result;
})
);
// Promise.all preserves input order regardless of completion order.
const errors = scans.flatMap((scan) => scan.errors);
const errors = [...rootErrors, ...scans.flatMap((scan) => scan.errors)];
// One directory reachable from two roots (a `cubeDirs` entry nested under a
// `.npcubes` marker, say) is one cube seen twice, not a collision.
@@ -172,7 +257,7 @@ export async function loadCubes(): Promise<LoadResult> {
// The map is still populated for the callers that only report; a duplicate
// is fatal, so which candidate landed here never reaches a deploy.
const [first] = group;
cubes[id] = new Cube(first.manifest, first.dir, first.deployScript);
cubes[id] = new Cube(first.manifest, first.dir, first.deployScript, first.source);
}
return { cubes, errors };
+108
View File
@@ -0,0 +1,108 @@
/**
* Resolving cube packages named in `cubePackages` to directories on disk.
* @module cubes/packages
*/
import fs from 'node:fs';
import { createRequire } from 'node:module';
import path from 'node:path';
import type { CubePackageRef } from '../nopy.config.js';
/** An installed cube package, located and validated. */
export interface CubePackage {
/** The name it was requested under. */
name: string;
/** Absolute path to the package root. */
root: string;
/** Absolute paths to its cube directories, from `nopy.cubes`. */
dirs: string[];
}
/**
* Finds a package root without going through its `exports` map.
*
* `exports` is deliberately bypassed: a cube bundle ships directories, not an
* entry point, and requiring it to declare one would make the contract heavier
* for no gain. Reading `package.json` off disk also sidesteps pnpm's layout —
* `existsSync` follows the symlink pnpm plants at `node_modules/<name>`, which
* a directory scan would skip (`readdir` reports it as a symlink, not a
* directory).
*/
function findPackageRoot(ref: CubePackageRef): string | undefined {
// createRequire needs a file path, not a directory; the file need not exist.
const req = createRequire(path.join(ref.from, 'noop.js'));
for (const dir of req.resolve.paths(ref.spec) ?? []) {
if (fs.existsSync(path.join(dir, ref.spec, 'package.json'))) {
return path.join(dir, ref.spec);
}
}
return undefined;
}
/**
* Resolves every named package to its cube directories.
*
* Anything wrong is an error rather than a silent skip: naming a package in
* `cubePackages` is a statement that cubes are expected from it, and errors
* abort the run (see `nopy.main.ts`).
*/
export function resolveCubePackages(refs: CubePackageRef[]): {
packages: CubePackage[];
errors: string[];
} {
const packages: CubePackage[] = [];
const errors: string[] = [];
// `mergeValue` only de-duplicates arrays of primitives, and these are
// objects, so the same package named by a parent and a child config arrives
// twice. Last wins: configs merge root-first, so the last occurrence came
// from the most specific config and carries the right resolution origin.
const unique = new Map<string, CubePackageRef>();
for (const ref of refs) unique.set(ref.spec, ref);
for (const ref of unique.values()) {
const root = findPackageRoot(ref);
if (!root) {
errors.push(`Cube package '${ref.spec}' is not installed (looked up from ${ref.from}).`);
continue;
}
let manifest: { nopy?: { cubes?: unknown } };
try {
manifest = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf-8'));
} catch (err) {
errors.push(`Cube package '${ref.spec}': cannot read ${root}/package.json: ${err}`);
continue;
}
const declared = manifest.nopy?.cubes;
if (
!Array.isArray(declared) ||
declared.length === 0 ||
!declared.every((entry) => typeof entry === 'string')
) {
errors.push(
`Cube package '${ref.spec}' declares no cubes. ` +
`Expected "nopy": { "cubes": ["./cubes"] } in ${root}/package.json.`
);
continue;
}
const dirs: string[] = [];
for (const entry of declared as string[]) {
const dir = path.resolve(root, entry);
if (dir !== root && !dir.startsWith(root + path.sep)) {
errors.push(`Cube package '${ref.spec}': '${entry}' points outside the package.`);
} else if (!fs.existsSync(dir)) {
errors.push(`Cube package '${ref.spec}': '${entry}' does not exist in ${root}.`);
} else {
dirs.push(dir);
}
}
if (dirs.length > 0) packages.push({ name: ref.spec, root, dirs });
}
return { packages, errors };
}
+62
View File
@@ -0,0 +1,62 @@
/**
* A module resolve hook that lets a hand-written cube import the packages nopy
* itself already has.
*
* A manifest is loaded with `import(manifestPath)`, so its imports resolve from
* its own directory. A cube sitting in an arbitrary `cubeDirs` entry — no
* package.json above it, no node_modules beside it — therefore cannot import
* `@bitsquare/nopy-cube` or `zod` at all, and the run dies on
* ERR_MODULE_NOT_FOUND before a single deploy is built.
*
* A published cube bundle never reaches this: it declares its own dependencies
* and Node resolves them normally. This is for the local tree.
*
* Plain `.mjs` rather than TypeScript because the hook runs on its own thread,
* loaded by Node directly from `dist` — there is no compile step in that path.
*
* @module cubes/resolve-hook
*/
import { createRequire } from 'node:module';
import { pathToFileURL } from 'node:url';
/**
* The specifiers worth rescuing: what a manifest legitimately needs and cannot
* be expected to install for itself. Anything else stays a hard failure — a
* cube that wants a library should depend on it.
*/
const FALLBACK_ROOTS = ['@bitsquare/nopy-cube', '@bitsquare/nopy', 'zod'];
/** @type {NodeRequire | undefined} */
let fallbackRequire;
/**
* @param {{ from: string }} data - a URL inside the running CLI's own package,
* which is where the fallback resolution starts from.
*/
export function initialize(data) {
fallbackRequire = createRequire(data.from);
}
/** True for `zod` and for subpaths like `zod/v4` or `@bitsquare/nopy/package.json`. */
function isCovered(specifier) {
return FALLBACK_ROOTS.some((root) => specifier === root || specifier.startsWith(`${root}/`));
}
export async function resolve(specifier, context, next) {
try {
// Normal resolution first, always. A consumer that has its own copy
// installed keeps using it, so the hook can never introduce version skew —
// it only fills in for a lookup that was going to fail.
return await next(specifier, context);
} catch (error) {
if (!fallbackRequire || !isCovered(specifier)) throw error;
try {
return { url: pathToFileURL(fallbackRequire.resolve(specifier)).href, shortCircuit: true };
} catch {
// The CLI cannot see it either. Report the original failure, which names
// the importer rather than the CLI.
throw error;
}
}
}
-202
View File
@@ -1,202 +0,0 @@
/**
* Type definitions for Nopy cubes
* @module cubes/types
*/
import { z } from 'zod';
/**
* Any object schema, whatever its shape.
*
* Stands in for zod 3's `z.AnyZodObject`, which zod 4 removed.
*/
export type AnyObjectSchema = z.ZodObject<Record<string, z.ZodType<any>>>;
/**
* Variables that can be passed to a cube
*/
export type CubeVariables = Record<string, string | number | boolean>;
/**
* A dependency specification
*/
export type DependencySpec = string | [id: string, variables?: CubeVariables];
/**
* Context passed to cube hooks for executing other cubes
*/
export interface HookContext {
exec: (key: string, variables: CubeVariables) => Promise<void> | void;
}
/**
* Hook function type for before/after cube execution
*/
export type Hook<Schema extends AnyObjectSchema = AnyObjectSchema> = (
ctx: HookContext,
variables: z.infer<Schema>
) => void | Promise<void>;
/**
* User-defined specification for a cube
*/
export interface Manifest<Schema extends AnyObjectSchema = AnyObjectSchema> {
/** Unique identifier for the cube (used for dependency references) */
id: string;
/** Human-readable name of the cube */
name: string;
/** Zod schema for validating cube variables */
schema: Schema;
/** Dynamic dependency resolver based on collected variables */
dependencies?: (variables: z.infer<Schema>) => DependencySpec[];
/** Hooks to run before cube execution */
before?: Hook<Schema>[];
/** Hooks to run after cube execution */
after?: Hook<Schema>[];
}
/**
* Factory function and namespace for Manifest
*/
export function Manifest<Schema extends AnyObjectSchema>(
opts: Pick<Manifest<Schema>, 'name'> & Partial<Omit<Manifest<Schema>, 'name'>>
): Manifest<Schema> {
return {
id: opts.id ?? '',
name: opts.name,
schema: opts.schema ?? (z.object({}) as unknown as Schema),
dependencies: opts.dependencies,
before: opts.before ?? [],
after: opts.after ?? [],
};
}
export namespace Manifest {
/**
* Internal create helper
*/
export function create<Schema extends AnyObjectSchema>(
opts: Pick<Manifest<Schema>, 'name'> & Partial<Omit<Manifest<Schema>, 'name'>>
): Manifest<Schema> {
return Manifest(opts);
}
}
/**
* zod's runtime discriminant for a schema node, as a plain string.
*
* `instanceof z.ZodDefault` compares against the *running* copy of zod. A cube
* manifest is free to build its schema with a different copy — its own
* dependency, or one shipped inside a bundle — and then every `instanceof`
* quietly returns false and the caller falls through to a wrong answer instead
* of failing. `def.type` holds across instances, so nothing here may go back to
* `instanceof`.
*/
export function zodKind(zodType: unknown): string {
return (zodType as { def: { type: string } }).def.type;
}
/**
* The type a wrapper wraps — `.default()`, `.optional()`, `.nullable()`.
* Only call this for a node whose {@link zodKind} is one of those.
*/
export function zodInner(zodType: unknown): z.ZodType {
return (zodType as { def: { innerType: z.ZodType } }).def.innerType;
}
/**
* Reads the `.default()` off a schema field, unwrapping the wrappers that may
* sit above it (`.default().optional()`, `.default().nullable()`).
*
* Returns `undefined` for a field that declares no default — which is also how
* `requiredKeys()` recognises a field the user has to supply.
*/
function defaultValueOf(zodType: z.ZodType): unknown {
const kind = zodKind(zodType);
if (kind === 'default') {
// zod 4 exposes `defaultValue` as a getter that already invokes a lazily
// declared default; the function branch is insurance against that changing.
const { defaultValue } = (zodType as unknown as { def: { defaultValue: unknown } }).def;
return typeof defaultValue === 'function' ? defaultValue() : defaultValue;
}
if (kind === 'optional' || kind === 'nullable') {
return defaultValueOf(zodInner(zodType));
}
return undefined;
}
/**
* Where a cube was discovered.
*
* Worth carrying because a cube's own directory does not say how it got into
* the run: `/…/node_modules/@acme/cubes-net/cubes/x` could equally have come
* from a `cubeDirs` entry pointing straight at it.
*/
export type CubeSource =
/** Found under a `cubeDirs` entry or a `.npcubes` marker, at `dir`. */
| { type: 'dir'; dir: string }
/** Contributed by a package named in `cubePackages`. */
| { type: 'package'; packageName: string; dir: string };
/**
* A fully loaded cube with its filesystem location and runtime state
*/
export class Cube<Schema extends AnyObjectSchema = AnyObjectSchema> {
constructor(
public readonly manifest: Manifest<Schema>,
public readonly dir: string,
public readonly deployScript: string,
/** Defaults to the cube's own directory, for cubes built by hand. */
public readonly source: CubeSource = { type: 'dir', dir }
) {}
get id(): string {
return this.manifest.id;
}
get name(): string {
return this.manifest.name;
}
/**
* Returns default values for the cube's schema.
*
* Parsing an empty object resolves every default in one go, but it fails
* outright as soon as one field has no `.default()`. Falling back to a
* per-field read keeps the defaults that *are* declared instead of dropping
* the whole set — a single required field used to leave the cube with no
* variables at all.
*/
getDefaults(): z.infer<Schema> {
const parsed = this.manifest.schema.safeParse({});
if (parsed.success) return parsed.data as z.infer<Schema>;
const defaults: Record<string, unknown> = {};
for (const [key, zodType] of Object.entries(this.manifest.schema.shape)) {
const value = defaultValueOf(zodType);
if (value !== undefined) defaults[key] = value;
}
return defaults as z.infer<Schema>;
}
/**
* Schema keys that have to be supplied from somewhere: no `.default()`, and
* not optional. Nothing else can fill them in, so a run that cannot prompt
* has to fail rather than deploy a cube with the value missing.
*/
requiredKeys(): string[] {
return Object.entries(this.manifest.schema.shape)
.filter(([, zodType]) => !zodType.safeParse(undefined).success)
.map(([key]) => key);
}
}
/**
* Result of loading cubes from the filesystem
*/
export interface LoadResult {
/** Map of cube key to Cube object */
cubes: Record<string, Cube>;
/** List of errors encountered during loading */
errors: string[];
}
-38
View File
@@ -1,38 +0,0 @@
/**
* Utility functions for cubes
* @module cubes/utils
*/
/**
* Generates a random string of the specified length using the current nanotime as a seed.
*
* Uses a simple Linear Congruential Generator (LCG) seeded with high-resolution time.
* Suitable for generating unique identifiers, not for cryptographic purposes.
*
* @param length - The desired length of the random string (default: 5)
* @returns A random alphanumeric string of the specified length
*
* @example
* ```typescript
* const id = uniqid(); // e.g., "Kx7Pm"
* const longId = uniqid(10); // e.g., "Kx7PmQr2Yw"
* ```
*/
export function uniqid(length = 5): string {
const charset = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
const charsetLength = charset.length;
// Use process.hrtime.bigint() for high-resolution time in nanoseconds
let seed = Number(process.hrtime.bigint() % BigInt(Number.MAX_SAFE_INTEGER));
const randomString: string[] = [];
for (let i = 0; i < length; i++) {
// Simple linear congruential generator (LCG) for pseudo-randomness
seed = (seed * 48271) % 2147483647;
const index = seed % charsetLength;
randomString.push(charset[index]);
}
return randomString.join('');
}
+6 -8
View File
@@ -6,6 +6,9 @@
// Cubes module
export * from './cubes/index.js';
export type { Assignment, Origin, TVariables, Value } from './nopy.common.js';
// Variables
export { MASK, Variable, Variables } from './nopy.common.js';
export type {
ExecutionConfig,
HistoryConfig,
@@ -28,6 +31,8 @@ export type {
// Executor
export {
executeDeployCalls,
maskCommand,
maskVariables,
outputExecutionPlan,
summarizeResults,
} from './nopy.executor.js';
@@ -60,14 +65,7 @@ export {
} from './nopy.prompts.js';
export type { AuthSession, CubeSession, NopySession } from './nopy.session.js';
// Session management
export {
createSession,
filterInternalVariables,
listSessions,
loadSession,
saveSession,
separateEnvAndCubeVariables,
} from './nopy.session.js';
export { createSession, listSessions, loadSession, saveSession } from './nopy.session.js';
export type { WorkflowOptions, WorkflowResult } from './nopy.workflow.js';
// Workflow
export {
+188 -33
View File
@@ -1,49 +1,204 @@
/**
* Environment variable configuration
* Variable assignment and provenance
* @module nopy.common
*/
export type TVariables = Record<string, string | number | boolean>;
export namespace Variables {
export type ArtefactId = string;
export type Scope = 'defaults' | 'prompts' | 'params';
/** What a cube variable can hold — the value types `--data KEY=VALUE` can carry. */
export type Value = string | number | boolean;
/** A flat bag of variable values, keyed by name. */
export type TVariables = Record<string, Value>;
/**
* Where a value came from, in ascending precedence.
*
* The order is the point. It used to be implied by the field order of an object
* literal inside `Variables.get()` — load-bearing, invisible, and one careless
* reformat away from silently changing which value wins. Here it is stated once,
* in {@link RANK}, and everything else derives from it.
*
* - `default` — a `.default()` on the cube's schema
* - `env` — the `env` block of `.nopyrc.json`
* - `session` — read back from a recorded session on replay
* - `prompt` — what the user typed
* - `param` — handed over by a dependency spec or a hook's `exec()`
*/
export type Origin = 'default' | 'env' | 'session' | 'prompt' | 'param';
const RANK: Record<Origin, number> = {
default: 0,
env: 1,
session: 2,
prompt: 3,
param: 4,
};
/** One value handed to a variable, and where it came from. */
export interface Assignment {
value: Value;
origin: Origin;
}
export class Variables {
/** @summary env as configured in cube or session script */
defaults: Record<Variables.ArtefactId, TVariables> = {};
/** @summary env as configured via prompts */
prompts: Record<Variables.ArtefactId, TVariables> = {};
/** @summary env as handed via params (on hook calls) */
params: Record<Variables.ArtefactId, TVariables> = {};
/** What a secret shows as wherever a value would otherwise be printed. */
export const MASK = '********';
constructor(readonly global: TVariables = {}) {}
/**
* One variable of one cube, and every value it has ever been given.
*
* Two orderings are kept, deliberately: {@link assignments} is the raw trace in
* the order things happened, and {@link ordered} re-ranks it by origin. The
* first answers "how did we get here", the second answers "what wins".
*/
export class Variable {
/** Every assignment received, newest first. Never reordered. */
readonly assignments: Assignment[] = [];
assign(artefactId: Variables.ArtefactId, scope: Variables.Scope, values: TVariables = {}) {
if (!this[scope][artefactId]) {
this[scope][artefactId] = values;
} else {
Object.assign(this[scope][artefactId], values);
}
/**
* Declared a secret by the cube's manifest: kept out of saved sessions and
* masked wherever the value would otherwise be printed.
*/
redacted = false;
constructor(
readonly cube: string,
readonly name: string,
first: Assignment
) {
this.assign(first);
}
assign(assignment: Assignment): void {
this.assignments.unshift(assignment);
}
/**
* Merges the scopes for one cube, lowest precedence first:
* schema defaults → global `env` → prompts (or replayed session values) →
* params handed over by a dependency or a hook.
* The trace re-ranked by origin, winner first.
*
* Defaults sit at the bottom so `env` in `.nopyrc.json` can steer a run that
* never prompts (`--use-defaults`); a key that a dependency supplies is never
* prompted for, so prompts and params do not compete in practice.
* Stability is load-bearing here. The trace is newest-first and
* `Array.prototype.sort` is stable per spec, so two assignments sharing an
* origin keep their relative order and the newer one stays in front: the
* second dependency to pass a param wins, and the one it displaced is still
* visible underneath instead of being overwritten out of existence.
*/
get(artefactId: Variables.ArtefactId, scope?: Variables.Scope): TVariables {
if (scope) {
return this[scope][artefactId] || {};
}
get ordered(): Assignment[] {
return [...this.assignments].sort((a, b) => RANK[b.origin] - RANK[a.origin]);
}
/** The assignment that wins. Never undefined — a Variable is born with one. */
get effective(): Assignment {
return this.ordered[0];
}
get value(): Value {
return this.effective.value;
}
get origin(): Origin {
return this.effective.origin;
}
/** Safe to log: a redacted variable never yields its value. */
toJSON(): { cube: string; name: string; value: Value; origin: Origin } {
return {
...this.defaults[artefactId],
...this.global,
...this.prompts[artefactId],
...this.params[artefactId],
cube: this.cube,
name: this.name,
value: this.redacted ? MASK : this.value,
origin: this.origin,
};
}
}
/**
* Every variable of every cube in one run, with its provenance.
*/
export class Variables {
private readonly store: Record<string, Record<string, Variable>> = {};
private readonly secrets: Record<string, Set<string>> = {};
constructor(readonly env: TVariables = {}) {}
/**
* Marks keys of one cube as holding secrets.
*
* Retroactive as well as prospective, so it does not matter whether the
* caller declares before or after the values arrive.
*/
declareSecrets(cube: string, keys: readonly string[]): void {
this.secrets[cube] ??= new Set<string>();
const declared = this.secrets[cube];
for (const key of keys) declared.add(key);
for (const variable of this.all(cube)) {
if (declared.has(variable.name)) variable.redacted = true;
}
}
isSecret(cube: string, name: string): boolean {
return this.secrets[cube]?.has(name) ?? false;
}
/** Records values for one cube, all at the same origin. */
assign(cube: string, origin: Origin, values: TVariables = {}): void {
const bucket = this.bucket(cube);
for (const [name, value] of Object.entries(values)) {
const existing = bucket[name];
if (existing) existing.assign({ value, origin });
else bucket[name] = this.create(cube, name, { value, origin });
}
}
/** Every variable known for one cube. */
all(cube: string): Variable[] {
return Object.values(this.store[cube] ?? {});
}
/** One variable, or `undefined` if nothing has ever assigned to it. */
of(cube: string, name: string): Variable | undefined {
return this.store[cube]?.[name];
}
/** The effective values for one cube — what goes on the pyinfra command line. */
get(cube: string): TVariables {
const values: TVariables = {};
for (const variable of this.all(cube)) values[variable.name] = variable.value;
return values;
}
/**
* The effective values minus anything declared secret — what a session
* records. A secret is left out entirely rather than masked, so a replay sees
* it as absent and asks for it again.
*/
persistable(cube: string): TVariables {
const values: TVariables = {};
for (const variable of this.all(cube)) {
if (!variable.redacted) values[variable.name] = variable.value;
}
return values;
}
private create(cube: string, name: string, first: Assignment): Variable {
const variable = new Variable(cube, name, first);
variable.redacted = this.isSecret(cube, name);
return variable;
}
/**
* A cube's bucket, seeded on creation with the config `env`.
*
* `env` applies to every cube, so it becomes a real assignment on each of them
* rather than a parallel bag merged in at read time. That is what lets it
* carry an origin, show up in the trace, and lose to a prompt by the same rule
* as everything else.
*/
private bucket(cube: string): Record<string, Variable> {
const existing = this.store[cube];
if (existing) return existing;
const bucket: Record<string, Variable> = {};
this.store[cube] = bucket;
for (const [name, value] of Object.entries(this.env)) {
bucket[name] = this.create(cube, name, { value, origin: 'env' });
}
return bucket;
}
}
+47 -11
View File
@@ -3,9 +3,10 @@
* @module nopy.executor
*/
import type { DependencySpec } from '@bitsquare/nopy-cube';
import { getLogger } from '@logtape/logtape';
import { execa } from 'execa';
import type { DependencySpec } from './cubes/types.js';
import { MASK } from './nopy.common.js';
const log = getLogger(['nopy', 'executor']);
@@ -23,10 +24,47 @@ export interface DeployCall {
command: string[];
/** Environment variables for the cube */
env: Record<string, unknown>;
/** Schema keys the cube's manifest declared as secrets */
secrets?: string[];
/** Cube dependencies */
dependencies: DependencySpec[];
}
/**
* The command as it is safe to show: the SSH password, and every `--data KEY=…`
* whose key the manifest declared a secret, have their values replaced.
*
* pyinfra takes its data on the command line, so the real values have to be in
* `call.command` — this is the last point before they would reach a log, a
* `--print-only` dump or a dry-run plan.
*/
export function maskCommand(call: DeployCall): string {
const command = call.command.join(' ');
const quoteMeta = (key: string) => key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
// The builder always quotes a `--data` value, so the closing quote bounds it.
const masked = (call.secrets ?? []).reduce(
(acc, key) => acc.replace(new RegExp(`(--data "${quoteMeta(key)}=)[^"]*"`, 'g'), `$1${MASK}"`),
command
);
return masked.replace(/(--password )\S+/g, `$1${MASK}`);
}
/**
* The cube's variables as they are safe to show.
*
* This used to guess, masking any key whose name contained "password" — which
* missed `TOKEN` and `PSK`, and was defeated anyway by the unmasked command
* printed on the line above it. The manifest says which keys are secret now.
*/
export function maskVariables(call: DeployCall): Record<string, string> {
const secrets = new Set(call.secrets ?? []);
return Object.fromEntries(
Object.entries(call.env).map(([key, value]) => [key, secrets.has(key) ? MASK : String(value)])
);
}
/**
* Result of executing a deployment command
*/
@@ -73,7 +111,7 @@ async function executeCall(call: DeployCall): Promise<ExecutionResult> {
try {
log.info(`Executing: ${call.cube} -> ${call.host}`);
log.debug(`Command: ${commandStr}`);
log.debug(`Command: ${maskCommand(call)}`);
// Inherit stdio for live output
await execa({ shell: true })(commandStr, {
@@ -112,8 +150,8 @@ export function outputExecutionPlan(calls: DeployCall[], asJson?: boolean): void
const plan = calls.map((call) => ({
cube: call.cube,
host: call.host,
command: call.command.join(' '),
variables: call.env,
command: maskCommand(call),
variables: maskVariables(call),
}));
console.log(JSON.stringify({ plan }, null, 2));
return;
@@ -124,15 +162,13 @@ export function outputExecutionPlan(calls: DeployCall[], asJson?: boolean): void
for (let i = 0; i < calls.length; i++) {
const call = calls[i];
console.log(`Step ${i + 1}: ${call.cube} -> ${call.host}`);
console.log(` Command: ${call.command.join(' ')}`);
console.log(` Command: ${maskCommand(call)}`);
const envKeys = Object.keys(call.env);
if (envKeys.length > 0) {
const variables = maskVariables(call);
if (Object.keys(variables).length > 0) {
console.log(' Variables:');
for (const [key, value] of Object.entries(call.env)) {
// Mask sensitive values
const displayValue = key.toLowerCase().includes('password') ? '********' : String(value);
console.log(` ${key}=${displayValue}`);
for (const [key, value] of Object.entries(variables)) {
console.log(` ${key}=${value}`);
}
}
console.log();
+11 -3
View File
@@ -8,7 +8,12 @@ import { BuildContext } from './cubes/dependencies.js';
import { loadCubes } from './cubes/index.js';
import { Variables } from './nopy.common.js';
import { getConfigPaths, loadConfig } from './nopy.config.js';
import { type ExecutionResult, executeDeployCalls, summarizeResults } from './nopy.executor.js';
import {
type ExecutionResult,
executeDeployCalls,
maskCommand,
summarizeResults,
} from './nopy.executor.js';
import { addToHistory, DEFAULT_HISTORY_SIZE } from './nopy.history.js';
import { type NopySession, saveSession } from './nopy.session.js';
import { runWorkflow } from './nopy.workflow.js';
@@ -72,6 +77,9 @@ function printActiveConfig(
if (config.hosts.length > 0) lines.push(` Hosts: ${config.hosts.join(', ')}`);
if (config.cubeDirs.length > 0) lines.push(` Cube dirs: ${config.cubeDirs.join(', ')}`);
if (config.cubePackages.length > 0) {
lines.push(` Cube pkgs: ${config.cubePackages.map((ref) => ref.spec).join(', ')}`);
}
if (opts.continueOnError) lines.push(' Execution: continue-on-error');
const envEntries = Object.entries(config.env);
@@ -185,7 +193,7 @@ export async function nopy(opts: NopyOptions = {}): Promise<NopyResult | undefin
const sessionForSaving: NopySession = {
...workflow.session,
cubes: context.cubeSessions,
env: variables.get('global'),
env: config.env,
};
if (saveSessionPath && !workflow.isReplay) {
@@ -203,7 +211,7 @@ export async function nopy(opts: NopyOptions = {}): Promise<NopyResult | undefin
console.log('\n Deploy Commands\n ───────────────\n');
for (const call of context.deployCalls) {
console.log(` # ${call.cube} -> ${call.host}`);
console.log(` ${call.command.join(' ')}\n`);
console.log(` ${maskCommand(call)}\n`);
}
return {
success: true,
+28 -11
View File
@@ -36,11 +36,17 @@ function suggestCubes(input: string | undefined, choices: CubeChoice[]): CubeCho
export async function CubeSelection(
cubes: Record<string, Cube>
): Promise<{ selectedCubes: string[] }> {
// The package a cube came from is part of the label rather than a separate
// column: `suggest` filters on the label, so typing a package name narrows
// the list to that bundle.
const cubeChoices: CubeChoice[] = Object.values(cubes)
.sort((a, b) => a.id.localeCompare(b.id))
.map((cube) => ({
name: cube.id,
message: `${cube.id} - ${cube.name}`,
message:
cube.source.type === 'package'
? `${cube.id} - ${cube.name} (${cube.source.packageName})`
: `${cube.id} - ${cube.name}`,
}));
// Clear terminal and move cursor to top
@@ -176,24 +182,35 @@ interface FormChoice {
initial: string;
}
/**
* Asks the user for a cube's variables and records the answers.
*
* Reads what to offer out of `variables`, so the caller is expected to have
* assigned the schema defaults first — which `BuildContext.resolveCube` does.
* Deliberately not falling back to `cube.getDefaults()` here: calling it a
* second time re-evaluates every lazily declared default, so a cube generating
* one would show a different value than the one the run had already recorded.
*/
export async function VariableAssignment<S extends AnyObjectSchema>(
cube: Cube<S>,
variables: Variables
variables: Variables,
opts: { keys?: string[] } = {}
) {
const schema = cube.manifest.schema.shape;
const defaults = cube.getDefaults() as Record<string, unknown>;
const params = variables.get(cube.id, 'params');
const resolved = variables.get(cube.id);
const variablesToConfigure: Record<string, unknown> = {};
// Every schema key is offered, not just the ones carrying a `.default()` — a
// field without one is precisely the field that has to be asked about. Keys a
// dependency or hook already supplied are left alone. The value shown is the
// Every schema key is offered by default, not just the ones carrying a
// `.default()` — a field without one is precisely the field that has to be
// asked about. `opts.keys` narrows that to a subset, which is how a replay
// asks only about the gaps it cannot fill itself.
//
// A key a dependency or hook supplied is left alone. The value shown is the
// one the run would otherwise use, so `env` from `.nopyrc.json` is visible
// (and editable) rather than silently overridden by whatever is typed.
for (const key of Object.keys(schema)) {
if (params[key] !== undefined) continue;
variablesToConfigure[key] = resolved[key] ?? defaults[key];
for (const key of opts.keys ?? Object.keys(schema)) {
if (variables.of(cube.id, key)?.origin === 'param') continue;
variablesToConfigure[key] = resolved[key];
}
if (Object.keys(variablesToConfigure).length === 0) return;
@@ -217,7 +234,7 @@ export async function VariableAssignment<S extends AnyObjectSchema>(
const zodType = schema[key];
coercedResult[key] = zodType ? coerceValue(value, zodType) : value;
}
variables.assign(cube.id, 'prompts', coercedResult);
variables.assign(cube.id, 'prompt', coercedResult);
} catch {
// User cancelled
}
-49
View File
@@ -195,52 +195,3 @@ export function createSession(params: {
env: params.env,
};
}
/**
* Filters out internal variables from cube variables
*
* Internal variables are those used by the prompts system
* and should not be saved in session files.
*
* @param variables - Variables object
* @returns Filtered variables without internal keys
*/
export function filterInternalVariables(
variables: Record<string, unknown>
): Record<string, unknown> {
const internalKeys = ['customize'];
const filtered: Record<string, unknown> = {};
for (const [key, value] of Object.entries(variables)) {
if (!internalKeys.includes(key)) {
filtered[key] = value;
}
}
return filtered;
}
/**
* Separates environment variables from cube-specific variables
*
* @param allVariables - All variables including env and cube-specific
* @param envVariables - Known environment variables from config
* @returns Object with separate env and cube variables
*/
export function separateEnvAndCubeVariables(
allVariables: Record<string, unknown>,
envVariables: Record<string, unknown>
): { env: Record<string, unknown>; cubeVars: Record<string, unknown> } {
const env: Record<string, unknown> = {};
const cubeVars: Record<string, unknown> = {};
for (const [key, value] of Object.entries(allVariables)) {
if (key in envVariables) {
env[key] = value;
} else {
cubeVars[key] = value;
}
}
return { env, cubeVars };
}