[wip] cubes packaging and distribution via registry

packages/nopy/src/cubes/index.ts

@@ -31,6 +31,8 @@ export type {
 export {
   Cube,
   Manifest,
+  zodInner,
+  zodKind,
 } from './types.js';
 
 // Utilities

packages/nopy/src/cubes/loader.ts

@@ -50,68 +50,131 @@ function extractCubeId(manifest: Manifest): string | undefined {
   return match ? match[1] : undefined;
 }
 
+/** A cube found on disk, before ids have been checked against each other. */
+interface CubeCandidate {
+  id: string;
+  manifest: Manifest;
+  dir: string;
+  deployScript: string;
+}
+
+/** What one root directory contributed. */
+interface ScanResult {
+  candidates: CubeCandidate[];
+  errors: string[];
+}
+
 /**
- * Loads all cubes from discovered cube directories.
+ * Walks one root, collecting every cube below it.
+ *
+ * Deliberately does not decide anything about ids: a duplicate is only visible
+ * once every root has been walked, and stopping the descent here would hide
+ * whatever sits below the offending directory.
  */
-export async function loadCubes(): Promise<LoadResult> {
-  const cubesFolders = findCubeDirectories();
-  const cubes: Record<string, Cube> = {};
-  const errors: string[] = [];
+async function scanDirectory(currentDir: string, result: ScanResult): Promise<void> {
+  const entries = (await fs.readdir(currentDir, { withFileTypes: true })).sort((a, b) =>
+    a.name.localeCompare(b.name)
+  );
 
-  async function scanDirectory(currentDir: string, baseDir: string): Promise<void> {
-    const entries = await fs.readdir(currentDir, { withFileTypes: true });
+  const files = entries.filter((e) => e.isFile());
+  const manifestFile = files.find(
+    (f) => f.name === 'manifest.mjs' || f.name.endsWith('.manifest.mjs')
+  );
+  const deployFile = files.find((f) => f.name === 'deploy.py' || f.name.endsWith('.deploy.py'));
 
-    const files = entries.filter((e) => e.isFile());
-    const manifestFile = files.find(
-      (f) => f.name === 'manifest.mjs' || f.name.endsWith('.manifest.mjs')
-    );
-    const deployFile = files.find((f) => f.name === 'deploy.py' || f.name.endsWith('.deploy.py'));
+  if (manifestFile && deployFile) {
+    const manifestPath = path.join(currentDir, manifestFile.name);
 
-    if (manifestFile && deployFile) {
-      const cubePath = currentDir;
-      const manifestPath = path.join(cubePath, manifestFile.name);
+    try {
+      const manifest = (await import(manifestPath)).default as Manifest;
 
-      try {
-        const manifest = (await import(manifestPath)).default as Manifest;
+      if (!manifest || typeof manifest !== 'object') {
+        result.errors.push(`Invalid manifest export in ${manifestPath}`);
+      } else if (!manifest.name) {
+        result.errors.push(`Invalid manifest format in ${manifestPath}: missing 'name'`);
+      } else {
+        const cubeId = extractCubeId(manifest) || path.basename(currentDir);
 
-        if (!manifest || typeof manifest !== 'object') {
-          errors.push(`Invalid manifest export in ${manifestPath}`);
-        } else if (!manifest.name) {
-          errors.push(`Invalid manifest format in ${manifestPath}: missing 'name'`);
-        } else {
-          const cubeId = extractCubeId(manifest) || path.basename(cubePath);
+        // Ensure basic properties
+        manifest.id = cubeId;
+        manifest.schema = manifest.schema ?? z.object({});
 
-          if (cubes[cubeId]) {
-            errors.push(`Duplicate cube id '${cubeId}'`);
-            return;
-          }
-
-          // Ensure basic properties
-          manifest.id = cubeId;
-          manifest.schema = manifest.schema ?? z.object({});
-
-          cubes[cubeId] = new Cube(manifest, cubePath, deployFile.name);
-        }
-      } catch (err) {
-        errors.push(`Failed to load manifest ${manifestPath}: ${err}`);
-      }
-    }
-
-    for (const entry of entries) {
-      if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
-        await scanDirectory(path.join(currentDir, entry.name), baseDir);
+        result.candidates.push({
+          id: cubeId,
+          manifest,
+          dir: currentDir,
+          deployScript: deployFile.name,
+        });
       }
+    } catch (err) {
+      result.errors.push(`Failed to load manifest ${manifestPath}: ${err}`);
     }
   }
 
-  await Promise.all(
+  for (const entry of entries) {
+    if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
+      await scanDirectory(path.join(currentDir, entry.name), result);
+    }
+  }
+}
+
+/** The message a duplicate id produces. Aborts the run — see `nopy.main.ts`. */
+function duplicateError(id: string, group: CubeCandidate[]): string {
+  const where = group.map((c) => `  ${c.dir}`).join('\n');
+  return (
+    `Duplicate cube id '${id}' from ${group.length} sources:\n${where}\n` +
+    `Rename one of them, or remove a source from .nopyrc.json.`
+  );
+}
+
+/**
+ * Loads all cubes from discovered cube directories.
+ *
+ * Scanning and id resolution are separate passes on purpose. Each root
+ * contributes its own candidate list, and those lists are concatenated in
+ * root order rather than in whichever order the concurrent scans happened to
+ * finish — so which cube is reported as "the duplicate" is the same on every
+ * run, which is what makes the hard error testable.
+ */
+export async function loadCubes(): Promise<LoadResult> {
+  const cubesFolders = findCubeDirectories();
+
+  const scans = await Promise.all(
     cubesFolders.map(async (folder) => {
+      const result: ScanResult = { candidates: [], errors: [] };
       if (fs.existsSync(folder)) {
-        await scanDirectory(folder, folder);
+        await scanDirectory(folder, result);
       }
+      return result;
     })
   );
 
+  // Promise.all preserves input order regardless of completion order.
+  const errors = scans.flatMap((scan) => scan.errors);
+
+  // One directory reachable from two roots (a `cubeDirs` entry nested under a
+  // `.npcubes` marker, say) is one cube seen twice, not a collision.
+  const seenDirs = new Set<string>();
+  const byId = new Map<string, CubeCandidate[]>();
+
+  for (const candidate of scans.flatMap((scan) => scan.candidates)) {
+    if (seenDirs.has(candidate.dir)) continue;
+    seenDirs.add(candidate.dir);
+
+    const group = byId.get(candidate.id);
+    if (group) group.push(candidate);
+    else byId.set(candidate.id, [candidate]);
+  }
+
+  const cubes: Record<string, Cube> = {};
+  for (const [id, group] of byId) {
+    if (group.length > 1) errors.push(duplicateError(id, group));
+    // The map is still populated for the callers that only report; a duplicate
+    // is fatal, so which candidate landed here never reaches a deploy.
+    const [first] = group;
+    cubes[id] = new Cube(first.manifest, first.dir, first.deployScript);
+  }
+
   return { cubes, errors };
 }
 

packages/nopy/src/cubes/types.ts

@@ -82,6 +82,28 @@ export namespace Manifest {
   }
 }
 
+/**
+ * zod's runtime discriminant for a schema node, as a plain string.
+ *
+ * `instanceof z.ZodDefault` compares against the *running* copy of zod. A cube
+ * manifest is free to build its schema with a different copy — its own
+ * dependency, or one shipped inside a bundle — and then every `instanceof`
+ * quietly returns false and the caller falls through to a wrong answer instead
+ * of failing. `def.type` holds across instances, so nothing here may go back to
+ * `instanceof`.
+ */
+export function zodKind(zodType: unknown): string {
+  return (zodType as { def: { type: string } }).def.type;
+}
+
+/**
+ * The type a wrapper wraps — `.default()`, `.optional()`, `.nullable()`.
+ * Only call this for a node whose {@link zodKind} is one of those.
+ */
+export function zodInner(zodType: unknown): z.ZodType {
+  return (zodType as { def: { innerType: z.ZodType } }).def.innerType;
+}
+
 /**
  * Reads the `.default()` off a schema field, unwrapping the wrappers that may
  * sit above it (`.default().optional()`, `.default().nullable()`).
@@ -90,16 +112,32 @@ export namespace Manifest {
  * `requiredKeys()` recognises a field the user has to supply.
  */
 function defaultValueOf(zodType: z.ZodType): unknown {
-  if (zodType instanceof z.ZodDefault) {
-    const { defaultValue } = zodType._def as { defaultValue: unknown };
+  const kind = zodKind(zodType);
+  if (kind === 'default') {
+    // zod 4 exposes `defaultValue` as a getter that already invokes a lazily
+    // declared default; the function branch is insurance against that changing.
+    const { defaultValue } = (zodType as unknown as { def: { defaultValue: unknown } }).def;
     return typeof defaultValue === 'function' ? defaultValue() : defaultValue;
   }
-  if (zodType instanceof z.ZodOptional || zodType instanceof z.ZodNullable) {
-    return defaultValueOf(zodType._def.innerType as z.ZodType);
+  if (kind === 'optional' || kind === 'nullable') {
+    return defaultValueOf(zodInner(zodType));
   }
   return undefined;
 }
 
+/**
+ * Where a cube was discovered.
+ *
+ * Worth carrying because a cube's own directory does not say how it got into
+ * the run: `/…/node_modules/@acme/cubes-net/cubes/x` could equally have come
+ * from a `cubeDirs` entry pointing straight at it.
+ */
+export type CubeSource =
+  /** Found under a `cubeDirs` entry or a `.npcubes` marker, at `dir`. */
+  | { type: 'dir'; dir: string }
+  /** Contributed by a package named in `cubePackages`. */
+  | { type: 'package'; packageName: string; dir: string };
+
 /**
  * A fully loaded cube with its filesystem location and runtime state
  */
@@ -107,7 +145,9 @@ export class Cube<Schema extends AnyObjectSchema = AnyObjectSchema> {
   constructor(
     public readonly manifest: Manifest<Schema>,
     public readonly dir: string,
-    public readonly deployScript: string
+    public readonly deployScript: string,
+    /** Defaults to the cube's own directory, for cubes built by hand. */
+    public readonly source: CubeSource = { type: 'dir', dir }
   ) {}
 
   get id(): string {

packages/nopy/src/nopy.config.ts

@@ -55,13 +55,41 @@ export type ResolutionConfig = {
 };
 
 /**
- * Raw config file structure (includes resolution)
+ * A cube package named in `cubePackages`, paired with where it was named.
+ *
+ * Node resolution has to start from the config file that asked for the package,
+ * not from `process.cwd()` — otherwise a package listed in `~/.nopyrc.json`
+ * only resolves in projects that happen to depend on it themselves. This is the
+ * same problem {@link PATH_PROPERTIES} solves for `cubeDirs`, except the answer
+ * is a reference to resolve later rather than a rewritten path.
  */
-export interface NopyConfigFile extends Partial<NopyConfig> {
+export interface CubePackageRef {
+  /** The package name as written in the config, e.g. `@acme/cubes-net`. */
+  spec: string;
+  /** Directory of the `.nopyrc.json` that named it. */
+  from: string;
+}
+
+/**
+ * Raw config file structure (includes resolution)
+ *
+ * Diverges from {@link NopyConfig} for `cubePackages`: a file lists plain
+ * package names, and loading turns each into a {@link CubePackageRef}.
+ */
+export interface NopyConfigFile extends Omit<Partial<NopyConfig>, 'cubePackages'> {
+  /** Cube packages to load, by package name */
+  cubePackages?: string[];
   /** Customize merge behavior for specific properties */
   resolution?: ResolutionConfig;
 }
 
+/**
+ * A config file whose paths have been resolved — what actually gets merged.
+ */
+type ResolvedConfigFile = Omit<NopyConfigFile, 'cubePackages'> & {
+  cubePackages?: CubePackageRef[];
+};
+
 /**
  * Nopy configuration file structure
  */
@@ -70,6 +98,8 @@ export interface NopyConfig {
   hosts: string[];
   /** Directories to search for cubes */
   cubeDirs: string[];
+  /** Installed packages to load cubes from */
+  cubePackages: CubePackageRef[];
   /** Global environment variables */
   env: TVariables;
   /** Logging configuration */
@@ -86,6 +116,7 @@ export interface NopyConfig {
 const DEFAULT_CONFIG: NopyConfig = {
   hosts: [],
   cubeDirs: [],
+  cubePackages: [],
   env: {},
 };
 
@@ -219,30 +250,33 @@ const PATH_PROPERTIES: (keyof NopyConfig)[] = ['cubeDirs'];
  * Resolves relative paths in a config file based on its location
  * Only resolves paths for properties that are known to contain filesystem paths
  */
-function resolveConfigPaths(config: NopyConfigFile, configPath: string): NopyConfigFile {
+function resolveConfigPaths(config: NopyConfigFile, configPath: string): ResolvedConfigFile {
   const configDir = path.dirname(configPath);
-  const resolved: NopyConfigFile = {};
+  const resolved: Record<string, unknown> = {};
 
   for (const [key, value] of Object.entries(config)) {
     if (key === 'resolution') {
       // Don't resolve the resolution config itself
       resolved[key] = value as ResolutionConfig;
+    } else if (key === 'cubePackages') {
+      // Not a path — a package name, tagged with where to resolve it from.
+      resolved[key] = (value as string[]).map((spec) => ({ spec, from: configDir }));
     } else if (PATH_PROPERTIES.includes(key as keyof NopyConfig)) {
       // Only resolve paths for known path properties
-      resolved[key as keyof NopyConfigFile] = resolveRelativePaths(value, configDir) as any;
+      resolved[key] = resolveRelativePaths(value, configDir);
     } else {
       // Copy other properties as-is (including hosts)
-      resolved[key as keyof NopyConfigFile] = value as any;
+      resolved[key] = value;
     }
   }
 
-  return resolved;
+  return resolved as ResolvedConfigFile;
 }
 
 /**
  * Merges a child config into a parent config
  */
-function mergeConfigs(parent: NopyConfig, childFile: NopyConfigFile): NopyConfig {
+function mergeConfigs(parent: NopyConfig, childFile: ResolvedConfigFile): NopyConfig {
   const resolution = childFile.resolution || {};
   const result: Record<string, unknown> = { ...parent };
 

packages/nopy/src/nopy.prompts.ts

@@ -6,8 +6,8 @@
 import Enquirer from 'enquirer';
 import fuzzy from 'fuzzy';
 import inquirer from 'inquirer';
-import { z } from 'zod';
-import type { AnyObjectSchema, Cube } from './cubes/index.js';
+import type { z } from 'zod';
+import { type AnyObjectSchema, type Cube, zodInner, zodKind } from './cubes/index.js';
 import type { Variables } from './nopy.common.js';
 
 interface CubeChoice {
@@ -142,20 +142,32 @@ export async function HostSelection(hosts: string[]): Promise<string> {
   return selectedHost.customHost ?? selectedHost.host;
 }
 
+/**
+ * Turns a form answer — always a string — back into what the schema declares.
+ *
+ * Discriminates on {@link zodKind} rather than `instanceof`: the schema may
+ * have been built by a copy of zod that is not the one this file imported, and
+ * `instanceof` would then fail open and leave every value a string.
+ */
 function coerceValue(value: unknown, zodType: z.core.$ZodType): unknown {
   if (typeof value !== 'string') return value;
-  if (zodType instanceof z.ZodDefault) return coerceValue(value, zodType._def.innerType);
-  if (zodType instanceof z.ZodOptional) return coerceValue(value, zodType._def.innerType);
-  if (zodType instanceof z.ZodNullable) {
-    if (value === 'null' || value === '') return null;
-    return coerceValue(value, zodType._def.innerType);
+
+  switch (zodKind(zodType)) {
+    case 'default':
+    case 'optional':
+      return coerceValue(value, zodInner(zodType));
+    case 'nullable':
+      if (value === 'null' || value === '') return null;
+      return coerceValue(value, zodInner(zodType));
+    case 'boolean':
+      return value === 'true' || value === 'yes' || value === '1';
+    case 'number': {
+      const num = Number(value);
+      return Number.isNaN(num) ? value : num;
+    }
+    default:
+      return value;
   }
-  if (zodType instanceof z.ZodBoolean) return value === 'true' || value === 'yes' || value === '1';
-  if (zodType instanceof z.ZodNumber) {
-    const num = Number(value);
-    return Number.isNaN(num) ? value : num;
-  }
-  return value;
 }
 
 interface FormChoice {