mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-04 06:29:42 +02:00
feat(docker): export/import (move a container to another machine) + boot reaper
- src/docker-export.ts: full-image export (pause-consistent commit + save|stream + workspace tar + manifest -> one .codeman-container.tgz) and workspace-only; import validates manifest + per-member sha256, traversal-guards the workspace tar, docker load + quarantine re-tag (never overwrites a local tag). Bounded by runWithConversionLimit; free-space precheck; docker rmi in finally; sealed containers refuse full-image export. - routes: POST /api/docker-cases/:name/export (background + SSE), GET/DELETE /api/docker-exports, GET download, POST /api/docker-cases/import (-> new host+case) - instance-scoped boot reaper (docker-hosts.reapOrphanedDockerContainers) wired after restoreMuxSessions; never touches another instance's containers - SSE docker:exportComplete/exportFailed/importComplete (both registries) - fix: stream pipeline in saveImageToTar so the bundle isn't truncated VERIFIED end-to-end on real docker: full export -> 326MB valid bundle -> delete case -> import -> new container runs from the quarantined image with the workspace file AND the in-image change both restored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer
@@ -0,0 +1,416 @@
|
||||
/**
|
||||
* @fileoverview Docker case export / import: move a container (toolchain + any
|
||||
* in-image changes) PLUS its workspace to another machine as one portable
|
||||
* `.codeman-container.tgz`, and restore it.
|
||||
*
|
||||
* A full-image export = `docker commit` the running container to an image ->
|
||||
* `docker save` that image -> tar the bind-mounted workspace -> a manifest, all
|
||||
* bundled into one gzip tarball. A workspace-only export skips the image (fast,
|
||||
* files-only). Import validates the manifest + per-member checksums, extracts the
|
||||
* workspace with a path-traversal guard, `docker load`s the image and RE-TAGS it
|
||||
* into a quarantined namespace (never overwriting a local tag), and hands the
|
||||
* caller enough to recreate a hardened case on the destination.
|
||||
*
|
||||
* Safety (all from the design critic): pause the container spanning the workspace
|
||||
* tar AND the commit so the two artifacts are mutually consistent; a free-space
|
||||
* precheck (a full docker graph wedges EVERY session on the host); `docker rmi`
|
||||
* the intermediate image in a finally; sealed containers refuse a full-image
|
||||
* export (an in-container login would ride the committed layer); import rejects
|
||||
* absolute / `..` tar members and checksum mismatches. Bounded by
|
||||
* runWithConversionLimit so N exports cannot fork-bomb the host.
|
||||
*
|
||||
* @module docker-export
|
||||
*/
|
||||
|
||||
import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import { join, basename } from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { DockerEngine, SessionDocker } from './types.js';
|
||||
import { runWithConversionLimit } from './document-conversion-limiter.js';
|
||||
|
||||
const IS_TEST_MODE = !!process.env.VITEST;
|
||||
|
||||
/** Refuse to export when the target filesystem has less than this free (a full graph wedges the daemon). */
|
||||
export const DOCKER_EXPORT_MIN_FREE_BYTES = 2 * 1024 * 1024 * 1024; // 2 GiB
|
||||
|
||||
/** Manifest schema version (bump on any breaking field change). */
|
||||
export const DOCKER_EXPORT_SCHEMA = 1;
|
||||
|
||||
export type DockerExportMode = 'full' | 'workspace';
|
||||
|
||||
export interface DockerExportManifest {
|
||||
schemaVersion: number;
|
||||
caseName: string;
|
||||
mode: DockerExportMode;
|
||||
engine: DockerEngine;
|
||||
image: string;
|
||||
containerWorkdir: string;
|
||||
network: string;
|
||||
createdAt: number;
|
||||
codemanVersion: string;
|
||||
mountCredentials: boolean;
|
||||
/** True when the bundle provably carries no credentials (convenient-mode workspace, or a full image whose creds were bind-mounted and thus never committed). */
|
||||
secretFree: boolean;
|
||||
/** sha256 of each bundle member that is present. */
|
||||
checksums: { image?: string; workspace?: string };
|
||||
}
|
||||
|
||||
// ========== Pure helpers (unit-tested) ==========
|
||||
|
||||
/** Raw argv prefix for the engine (NO shell escaping — used with spawn). */
|
||||
export function dockerArgv(docker: Pick<SessionDocker, 'engine' | 'context' | 'daemonHost'>): string[] {
|
||||
const argv: string[] = [docker.engine === 'podman' ? 'podman' : 'docker'];
|
||||
if (docker.context) argv.push('--context', docker.context);
|
||||
if (docker.daemonHost) argv.push('-H', docker.daemonHost);
|
||||
return argv;
|
||||
}
|
||||
|
||||
/** Portable bundle filename for a case export. */
|
||||
export function exportBundleName(caseName: string, timestamp: number, mode: DockerExportMode): string {
|
||||
const suffix = mode === 'workspace' ? 'workspace' : 'container';
|
||||
return `${caseName}-${timestamp}.codeman-${suffix}.tgz`;
|
||||
}
|
||||
|
||||
/** Quarantined image tag for an imported bundle (never overwrites a local tag). */
|
||||
export function importedImageTag(caseName: string, timestamp: number): string {
|
||||
return `codeman/imported-${caseName}:${timestamp}`;
|
||||
}
|
||||
|
||||
/** Intermediate commit tag for a full-image export (unique per export, rmi'd in finally). */
|
||||
export function exportImageTag(caseName: string, timestamp: number): string {
|
||||
return `codeman/export-${caseName}:${timestamp}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject a tar member path that would escape the extraction root (absolute path
|
||||
* or a `..` component). The import-side traversal guard.
|
||||
*/
|
||||
export function isSafeTarMember(member: string): boolean {
|
||||
const trimmed = member.trim();
|
||||
if (!trimmed || trimmed === './') return true;
|
||||
if (trimmed.startsWith('/')) return false;
|
||||
// Normalize separators and check each component.
|
||||
return !trimmed.split('/').some((part) => part === '..');
|
||||
}
|
||||
|
||||
/** Parse the image id/ref from `docker load` output ("Loaded image: x" / "Loaded image ID: sha256:..."). */
|
||||
export function parseLoadedImageRef(loadOutput: string): string | null {
|
||||
const idMatch = loadOutput.match(/Loaded image ID:\s*(sha256:[0-9a-f]+)/i);
|
||||
if (idMatch) return idMatch[1];
|
||||
const refMatch = loadOutput.match(/Loaded image:\s*(\S+)/i);
|
||||
if (refMatch) return refMatch[1];
|
||||
return null;
|
||||
}
|
||||
|
||||
// ========== IO helpers ==========
|
||||
|
||||
function run(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
opts: { timeout?: number } = {}
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
if (opts.timeout) {
|
||||
timer = setTimeout(() => {
|
||||
child.kill('SIGKILL');
|
||||
reject(new Error(`${cmd} timed out after ${opts.timeout}ms`));
|
||||
}, opts.timeout);
|
||||
}
|
||||
child.stdout.on('data', (d) => (stdout += d));
|
||||
child.stderr.on('data', (d) => (stderr += d));
|
||||
child.on('error', (err) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
reject(err);
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (code === 0) resolve({ stdout, stderr });
|
||||
else reject(new Error(`${cmd} ${args.join(' ')} exited ${code}: ${stderr.trim()}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream `docker save <tag>` stdout to a raw tar file (no shell, no double-gzip).
|
||||
* Uses stream `pipeline` so completion means the write stream is FULLY flushed to
|
||||
* disk (a naive child 'close' resolves before the last chunks land, truncating the
|
||||
* file — a real bug caught in end-to-end testing), AND waits for a clean exit code.
|
||||
*/
|
||||
async function saveImageToTar(argv: string[], tag: string, outPath: string): Promise<void> {
|
||||
const child = spawn(argv[0], [...argv.slice(1), 'save', tag], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stderr = '';
|
||||
child.stderr.on('data', (d) => (stderr += d));
|
||||
const exited = new Promise<void>((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('close', (code) =>
|
||||
code === 0 ? resolve() : reject(new Error(`docker save exited ${code}: ${stderr.trim()}`))
|
||||
);
|
||||
});
|
||||
// pipeline resolves only after the destination has fully flushed.
|
||||
await Promise.all([pipeline(child.stdout, createWriteStream(outPath)), exited]);
|
||||
}
|
||||
|
||||
async function sha256File(path: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('sha256');
|
||||
const stream = createReadStream(path);
|
||||
stream.on('data', (d) => hash.update(d));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
});
|
||||
}
|
||||
|
||||
async function freeBytes(path: string): Promise<number> {
|
||||
try {
|
||||
const stat = await fs.statfs(path);
|
||||
return Number(stat.bavail) * Number(stat.bsize);
|
||||
} catch {
|
||||
return Number.POSITIVE_INFINITY; // statfs unsupported — don't block
|
||||
}
|
||||
}
|
||||
|
||||
async function isContainerRunning(argv: string[], container: string): Promise<boolean> {
|
||||
try {
|
||||
const { stdout } = await run(argv[0], [...argv.slice(1), 'inspect', '-f', '{{.State.Running}}', container], {
|
||||
timeout: 15_000,
|
||||
});
|
||||
return stdout.trim() === 'true';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExportResult {
|
||||
bundlePath: string;
|
||||
manifest: DockerExportManifest;
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a docker case to a portable bundle. Bounded by runWithConversionLimit.
|
||||
* `full` mode commits + saves the image AND tars the workspace; `workspace` mode
|
||||
* tars just the workspace. The container is paused across the artifact capture so
|
||||
* image and workspace are mutually consistent.
|
||||
*/
|
||||
export async function exportDockerCase(params: {
|
||||
docker: SessionDocker;
|
||||
caseName: string;
|
||||
timestamp: number;
|
||||
exportsDir: string;
|
||||
mode: DockerExportMode;
|
||||
codemanVersion: string;
|
||||
}): Promise<ExportResult> {
|
||||
const { docker, caseName, timestamp, exportsDir, mode, codemanVersion } = params;
|
||||
|
||||
if (mode === 'full' && !docker.mountCredentials) {
|
||||
throw new Error(
|
||||
'full-image export is refused for a sealed (mountCredentials:false) container: an in-container login would ride the committed image layer. Use a workspace-only export.'
|
||||
);
|
||||
}
|
||||
|
||||
if (IS_TEST_MODE) {
|
||||
// No real docker/tar under vitest — return a deterministic stub.
|
||||
const manifest: DockerExportManifest = {
|
||||
schemaVersion: DOCKER_EXPORT_SCHEMA,
|
||||
caseName,
|
||||
mode,
|
||||
engine: docker.engine,
|
||||
image: docker.image,
|
||||
containerWorkdir: docker.containerWorkdir,
|
||||
network: docker.network,
|
||||
createdAt: timestamp,
|
||||
codemanVersion,
|
||||
mountCredentials: docker.mountCredentials,
|
||||
secretFree: true,
|
||||
checksums: {},
|
||||
};
|
||||
return { bundlePath: join(exportsDir, exportBundleName(caseName, timestamp, mode)), manifest, sizeBytes: 0 };
|
||||
}
|
||||
|
||||
return runWithConversionLimit(async () => {
|
||||
if (!existsSync(exportsDir)) mkdirSync(exportsDir, { recursive: true });
|
||||
|
||||
const free = await freeBytes(exportsDir);
|
||||
if (free < DOCKER_EXPORT_MIN_FREE_BYTES) {
|
||||
throw new Error(
|
||||
`not enough free space to export (need >= ${Math.round(DOCKER_EXPORT_MIN_FREE_BYTES / 1e9)}GB, have ${Math.round(free / 1e9)}GB). A full docker graph wedges every session on the host.`
|
||||
);
|
||||
}
|
||||
|
||||
const argv = dockerArgv(docker);
|
||||
const bundlePath = join(exportsDir, exportBundleName(caseName, timestamp, mode));
|
||||
const stageDir = join(exportsDir, `.stage-${caseName}-${timestamp}`);
|
||||
mkdirSync(stageDir, { recursive: true });
|
||||
const wasRunning = await isContainerRunning(argv, docker.containerName);
|
||||
let commitTag: string | undefined;
|
||||
|
||||
try {
|
||||
if (wasRunning) {
|
||||
await run(argv[0], [...argv.slice(1), 'pause', docker.containerName], { timeout: 30_000 }).catch(() => {});
|
||||
}
|
||||
|
||||
const checksums: DockerExportManifest['checksums'] = {};
|
||||
|
||||
if (mode === 'full') {
|
||||
commitTag = exportImageTag(caseName, timestamp);
|
||||
// Blank instance-specific committed env so the image carries no stale host refs.
|
||||
await run(
|
||||
argv[0],
|
||||
[
|
||||
...argv.slice(1),
|
||||
'commit',
|
||||
'-c',
|
||||
'ENV CODEMAN_API_URL=',
|
||||
'-c',
|
||||
'ENV CODEMAN_HOOK_SECRET_FILE=',
|
||||
docker.containerName,
|
||||
commitTag,
|
||||
],
|
||||
{ timeout: 300_000 }
|
||||
);
|
||||
const imageTar = join(stageDir, 'image.tar');
|
||||
await saveImageToTar(argv, commitTag, imageTar);
|
||||
checksums.image = await sha256File(imageTar);
|
||||
}
|
||||
|
||||
const workspaceTar = join(stageDir, 'workspace.tar');
|
||||
await run('tar', ['-cf', workspaceTar, '-C', docker.hostWorkspacePath, '.'], { timeout: 300_000 });
|
||||
checksums.workspace = await sha256File(workspaceTar);
|
||||
|
||||
const manifest: DockerExportManifest = {
|
||||
schemaVersion: DOCKER_EXPORT_SCHEMA,
|
||||
caseName,
|
||||
mode,
|
||||
engine: docker.engine,
|
||||
image: docker.image,
|
||||
containerWorkdir: docker.containerWorkdir,
|
||||
network: docker.network,
|
||||
createdAt: timestamp,
|
||||
codemanVersion,
|
||||
mountCredentials: docker.mountCredentials,
|
||||
// Convenient mode keeps creds on bind mounts (never committed), so the bundle is secret-free.
|
||||
secretFree: docker.mountCredentials,
|
||||
checksums,
|
||||
};
|
||||
await fs.writeFile(join(stageDir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
||||
|
||||
const members =
|
||||
mode === 'full' ? ['manifest.json', 'image.tar', 'workspace.tar'] : ['manifest.json', 'workspace.tar'];
|
||||
await run('tar', ['-czf', bundlePath, '-C', stageDir, ...members], { timeout: 300_000 });
|
||||
|
||||
const stat = await fs.stat(bundlePath);
|
||||
return { bundlePath, manifest, sizeBytes: stat.size };
|
||||
} finally {
|
||||
// Always remove the intermediate image + stage dir, and unpause.
|
||||
if (commitTag) {
|
||||
await run(argv[0], [...argv.slice(1), 'rmi', commitTag], { timeout: 60_000 }).catch(() => {});
|
||||
}
|
||||
await fs.rm(stageDir, { recursive: true, force: true }).catch(() => {});
|
||||
if (wasRunning) {
|
||||
await run(argv[0], [...argv.slice(1), 'unpause', docker.containerName], { timeout: 30_000 }).catch(() => {});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
manifest: DockerExportManifest;
|
||||
/** Quarantined image ref the destination case should use (full mode only). */
|
||||
importedImage?: string;
|
||||
/** Directory the workspace was extracted into. */
|
||||
workspacePath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a bundle produced by exportDockerCase: validate the manifest + per-member
|
||||
* checksums, extract the workspace (traversal-guarded) into destWorkspace, and, in
|
||||
* full mode, `docker load` the image and re-tag it into a quarantined namespace.
|
||||
*/
|
||||
export async function importDockerBundle(params: {
|
||||
bundlePath: string;
|
||||
destWorkspace: string;
|
||||
engine: DockerEngine;
|
||||
timestamp: number;
|
||||
}): Promise<ImportResult> {
|
||||
const { bundlePath, destWorkspace, engine, timestamp } = params;
|
||||
const argv: string[] = [engine === 'podman' ? 'podman' : 'docker'];
|
||||
|
||||
if (IS_TEST_MODE) {
|
||||
const raw = await fs.readFile(bundlePath, 'utf-8').catch(() => '{}');
|
||||
return { manifest: JSON.parse(raw) as DockerExportManifest, workspacePath: destWorkspace };
|
||||
}
|
||||
|
||||
const stageDir = `${destWorkspace}.import-stage-${timestamp}`;
|
||||
mkdirSync(stageDir, { recursive: true });
|
||||
try {
|
||||
await run('tar', ['-xzf', bundlePath, '-C', stageDir], { timeout: 300_000 });
|
||||
|
||||
const manifestRaw = await fs.readFile(join(stageDir, 'manifest.json'), 'utf-8');
|
||||
const manifest = JSON.parse(manifestRaw) as DockerExportManifest;
|
||||
if (manifest.schemaVersion !== DOCKER_EXPORT_SCHEMA) {
|
||||
throw new Error(`unsupported export schema version ${manifest.schemaVersion} (expected ${DOCKER_EXPORT_SCHEMA})`);
|
||||
}
|
||||
|
||||
// Integrity: verify checksums before trusting any member.
|
||||
const workspaceTar = join(stageDir, 'workspace.tar');
|
||||
if (manifest.checksums.workspace) {
|
||||
const actual = await sha256File(workspaceTar);
|
||||
if (actual !== manifest.checksums.workspace)
|
||||
throw new Error('workspace checksum mismatch (corrupt or tampered bundle)');
|
||||
}
|
||||
|
||||
// Traversal guard: reject absolute / `..` members before extraction.
|
||||
const { stdout: memberList } = await run('tar', ['-tf', workspaceTar], { timeout: 60_000 });
|
||||
for (const member of memberList.split('\n').filter(Boolean)) {
|
||||
if (!isSafeTarMember(member)) throw new Error(`unsafe path in workspace archive: ${member}`);
|
||||
}
|
||||
mkdirSync(destWorkspace, { recursive: true });
|
||||
await run('tar', ['--no-same-owner', '-xf', workspaceTar, '-C', destWorkspace], { timeout: 300_000 });
|
||||
|
||||
let importedImage: string | undefined;
|
||||
if (manifest.mode === 'full') {
|
||||
const imageTar = join(stageDir, 'image.tar');
|
||||
if (manifest.checksums.image) {
|
||||
const actual = await sha256File(imageTar);
|
||||
if (actual !== manifest.checksums.image)
|
||||
throw new Error('image checksum mismatch (corrupt or tampered bundle)');
|
||||
}
|
||||
const { stdout } = await run(argv[0], [...argv.slice(1), 'load', '-i', imageTar], { timeout: 300_000 });
|
||||
const loadedRef = parseLoadedImageRef(stdout);
|
||||
if (!loadedRef) throw new Error('could not determine loaded image ref');
|
||||
// Quarantine: re-tag by the loaded ref/id, never trusting the bundle's original tag.
|
||||
importedImage = importedImageTag(manifest.caseName, timestamp);
|
||||
await run(argv[0], [...argv.slice(1), 'tag', loadedRef, importedImage], { timeout: 60_000 });
|
||||
}
|
||||
|
||||
return { manifest, importedImage, workspacePath: destWorkspace };
|
||||
} finally {
|
||||
await fs.rm(stageDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/** List export bundles in the exports dir (newest first), with size + mtime. */
|
||||
export async function listDockerExports(
|
||||
exportsDir: string
|
||||
): Promise<Array<{ name: string; sizeBytes: number; mtimeMs: number }>> {
|
||||
if (!existsSync(exportsDir)) return [];
|
||||
const entries = await fs.readdir(exportsDir).catch(() => [] as string[]);
|
||||
const out: Array<{ name: string; sizeBytes: number; mtimeMs: number }> = [];
|
||||
for (const name of entries) {
|
||||
if (!name.endsWith('.tgz')) continue;
|
||||
try {
|
||||
const stat = await fs.stat(join(exportsDir, name));
|
||||
out.push({ name: basename(name), sizeBytes: stat.size, mtimeMs: stat.mtimeMs });
|
||||
} catch {
|
||||
/* skip */
|
||||
}
|
||||
}
|
||||
return out.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
||||
}
|
||||
Reference in New Issue
Block a user