mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
fix(attachments): harden document preview/thumbnail path (review of #120)
Follow-up hardening applied during review of PR #120, addressing the adversarial multi-agent findings: - fix(preview): render auto-detected (workspace, unregistered) DOCX/PPTX via the file-preview route and PDFs via file-raw in openFilePreview. Previously the Preview button fell through to file-content, dumping the binary Office/PDF bytes as mojibake, and the new file-preview route was unreachable dead code. (MAJOR: file-preview-route-unreachable-detected-office) - perf(convert): add a global converter-concurrency limiter (document-conversion-limiter.ts) wrapping every pdftoppm / soffice / powershell spawn, so N simultaneous preview/thumbnail requests can no longer fork unbounded converter processes. Default cap 3, CODEMAN_MAX_DOCUMENT_CONVERSIONS. (MAJOR: no-converter-concurrency-limit) - fix(cache): bound the converted-PDF disk cache with LRU-by-mtime eviction (pruneDocumentPreviewCache, default 100 files, CODEMAN_MAX_PREVIEW_CACHE_FILES), run after each successful conversion. Was unbounded. (MAJOR/MINOR: preview-cache-unbounded-disk-growth) Tests: document-conversion-limiter.test.ts, document-preview-cache-eviction.test.ts, and route coverage for the four new endpoints in routes/file-routes-preview-thumbnail.test.ts (closes the missing-route-test gap). Verified end-to-end against real pdftoppm (thumbnail render + concurrency cap). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* @fileoverview Global concurrency limiter for spawning external document
|
||||
* converters (pdftoppm / LibreOffice `soffice` / Word-COM `powershell.exe`).
|
||||
*
|
||||
* Without a cap, N simultaneous thumbnail/preview requests for *distinct*
|
||||
* documents fork N converter processes at once — each held open for up to the
|
||||
* multi-minute conversion timeout. That is a localhost resource-exhaustion
|
||||
* (fork-bomb-shaped) vector: a handful of large PDFs detected at once can pin
|
||||
* CPU and RAM. This module serializes converter spawns down to a small fixed
|
||||
* pool; excess spawns queue (FIFO) until a slot frees. The in-flight cache in
|
||||
* `document-preview-cache.ts` already de-dups *identical* inputs; this bounds
|
||||
* the *distinct* case the cache can't.
|
||||
*
|
||||
* Permit accounting transfers the slot directly to the next waiter on release
|
||||
* (rather than decrement-then-reacquire) so the active count can never exceed
|
||||
* the cap even under interleaved async resumption.
|
||||
*
|
||||
* NOT re-entrant: never call `runWithConversionLimit` from inside a task that is
|
||||
* already holding a slot — a nested acquire under a full pool would deadlock.
|
||||
* The converter call sites only ever acquire once per request (the office path
|
||||
* acquires for `soffice` and `pdftoppm` sequentially, not nested).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Max converter processes allowed to run concurrently across the whole process.
|
||||
* Override with CODEMAN_MAX_DOCUMENT_CONVERSIONS (clamped to >= 1).
|
||||
*/
|
||||
const MAX_CONCURRENT_DOCUMENT_CONVERSIONS = (() => {
|
||||
const raw = Number(process.env.CODEMAN_MAX_DOCUMENT_CONVERSIONS);
|
||||
return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : 3;
|
||||
})();
|
||||
|
||||
let active = 0;
|
||||
const waiters: Array<() => void> = [];
|
||||
|
||||
/** Test/diagnostic hook: converters currently holding a slot. */
|
||||
export function getActiveConversionCount(): number {
|
||||
return active;
|
||||
}
|
||||
|
||||
function acquire(): Promise<void> {
|
||||
if (active < MAX_CONCURRENT_DOCUMENT_CONVERSIONS) {
|
||||
active++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => waiters.push(resolve));
|
||||
}
|
||||
|
||||
function release(): void {
|
||||
const next = waiters.shift();
|
||||
if (next) {
|
||||
// Hand the slot straight to the next waiter — `active` stays at the cap.
|
||||
next();
|
||||
} else {
|
||||
active--;
|
||||
}
|
||||
}
|
||||
|
||||
/** Run `task` once a converter slot is free, releasing the slot afterward. */
|
||||
export async function runWithConversionLimit<T>(task: () => Promise<T>): Promise<T> {
|
||||
await acquire();
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
@@ -9,11 +9,22 @@ import { tmpdir } from 'node:os';
|
||||
import { basename, dirname, extname, join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { promisify } from 'node:util';
|
||||
import { runWithConversionLimit } from './document-conversion-limiter.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const OFFICE_CONVERSION_TIMEOUT_MS = 5 * 60_000;
|
||||
const DOCUMENT_PREVIEW_CACHE_DIR = join(tmpdir(), 'codeman-document-preview-cache');
|
||||
/**
|
||||
* Cap on persistent converted-PDF files kept in DOCUMENT_PREVIEW_CACHE_DIR.
|
||||
* The cache key embeds the source mtime, so every edit to a doc orphans its
|
||||
* prior PDF; without a cap the dir grows unbounded across long-running sessions.
|
||||
* Override with CODEMAN_MAX_PREVIEW_CACHE_FILES (clamped to >= 1).
|
||||
*/
|
||||
const MAX_PREVIEW_CACHE_FILES = (() => {
|
||||
const raw = Number(process.env.CODEMAN_MAX_PREVIEW_CACHE_FILES);
|
||||
return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : 100;
|
||||
})();
|
||||
function buildWordExportPdfScript(sourcePath: string, outputPath: string): string {
|
||||
return `
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -50,6 +61,40 @@ export function clearDocumentPreviewCache(): void {
|
||||
inFlightOfficeConversions.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort LRU-ish eviction for the persistent converted-PDF cache: keeps at
|
||||
* most MAX_PREVIEW_CACHE_FILES `*.pdf` files in `cacheDir`, deleting the oldest
|
||||
* by mtime once over the cap. Never throws — a pruning failure must not fail the
|
||||
* conversion that triggered it. Only `*.pdf` files are considered, so the
|
||||
* transient `work-*` mkdtemp dirs are ignored.
|
||||
*/
|
||||
export async function pruneDocumentPreviewCache(cacheDir: string): Promise<void> {
|
||||
try {
|
||||
const entries = await fs.readdir(cacheDir);
|
||||
const pdfs = entries.filter((name) => name.toLowerCase().endsWith('.pdf'));
|
||||
if (pdfs.length <= MAX_PREVIEW_CACHE_FILES) return;
|
||||
|
||||
const stats = await Promise.all(
|
||||
pdfs.map(async (name) => {
|
||||
const fullPath = join(cacheDir, name);
|
||||
try {
|
||||
const stat = await fs.stat(fullPath);
|
||||
return { fullPath, mtimeMs: stat.mtimeMs ?? 0 };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const sorted = stats.filter((s): s is { fullPath: string; mtimeMs: number } => s !== null);
|
||||
sorted.sort((a, b) => a.mtimeMs - b.mtimeMs); // oldest first
|
||||
const toRemove = sorted.slice(0, Math.max(0, sorted.length - MAX_PREVIEW_CACHE_FILES));
|
||||
await Promise.all(toRemove.map((entry) => fs.rm(entry.fullPath, { force: true }).catch(() => {})));
|
||||
} catch {
|
||||
// Best-effort: pruning must never break a conversion.
|
||||
}
|
||||
}
|
||||
|
||||
export async function getOfficePreviewPdfPath(filePath: string, extension: string): Promise<string | null> {
|
||||
const ext = extension.toLowerCase().replace(/^\./, '');
|
||||
if (ext !== 'docx' && ext !== 'pptx') return null;
|
||||
@@ -147,23 +192,26 @@ async function convertWordDocumentToCachedPdf(filePath: string, cachePath: strin
|
||||
const sourcePath = wslMountPathToWindowsPath(sourceCopyPath);
|
||||
if (!sourcePath) return null;
|
||||
|
||||
await execFileAsync(
|
||||
'powershell.exe',
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-ExecutionPolicy',
|
||||
'Bypass',
|
||||
'-EncodedCommand',
|
||||
encodePowerShellCommand(buildWordExportPdfScript(sourcePath, outputPath)),
|
||||
],
|
||||
{
|
||||
timeout: OFFICE_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}
|
||||
await runWithConversionLimit(() =>
|
||||
execFileAsync(
|
||||
'powershell.exe',
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-ExecutionPolicy',
|
||||
'Bypass',
|
||||
'-EncodedCommand',
|
||||
encodePowerShellCommand(buildWordExportPdfScript(sourcePath, outputPath)),
|
||||
],
|
||||
{
|
||||
timeout: OFFICE_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
if (await fileExists(cachePath)) {
|
||||
await pruneDocumentPreviewCache(dirname(cachePath));
|
||||
return cachePath;
|
||||
}
|
||||
|
||||
@@ -186,33 +234,37 @@ async function convertLibreOfficeDocumentToCachedPdf(filePath: string, cachePath
|
||||
let workDir: string | undefined;
|
||||
try {
|
||||
await fs.mkdir(DOCUMENT_PREVIEW_CACHE_DIR, { recursive: true });
|
||||
workDir = await fs.mkdtemp(join(DOCUMENT_PREVIEW_CACHE_DIR, 'work-'));
|
||||
const profileDir = join(workDir, 'profile');
|
||||
const outDir = await fs.mkdtemp(join(DOCUMENT_PREVIEW_CACHE_DIR, 'work-'));
|
||||
workDir = outDir;
|
||||
const profileDir = join(outDir, 'profile');
|
||||
await fs.mkdir(profileDir, { recursive: true });
|
||||
|
||||
await execFileAsync(
|
||||
'soffice',
|
||||
[
|
||||
'--headless',
|
||||
'--nologo',
|
||||
'--nofirststartwizard',
|
||||
`-env:UserInstallation=${pathToFileURL(profileDir).href}`,
|
||||
'--convert-to',
|
||||
'pdf',
|
||||
'--outdir',
|
||||
workDir,
|
||||
filePath,
|
||||
],
|
||||
{
|
||||
timeout: OFFICE_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}
|
||||
await runWithConversionLimit(() =>
|
||||
execFileAsync(
|
||||
'soffice',
|
||||
[
|
||||
'--headless',
|
||||
'--nologo',
|
||||
'--nofirststartwizard',
|
||||
`-env:UserInstallation=${pathToFileURL(profileDir).href}`,
|
||||
'--convert-to',
|
||||
'pdf',
|
||||
'--outdir',
|
||||
outDir,
|
||||
filePath,
|
||||
],
|
||||
{
|
||||
timeout: OFFICE_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
const converted = (await fs.readdir(workDir)).find((name) => name.toLowerCase().endsWith('.pdf'));
|
||||
const converted = (await fs.readdir(outDir)).find((name) => name.toLowerCase().endsWith('.pdf'));
|
||||
if (!converted) return null;
|
||||
|
||||
await fs.rename(join(workDir, converted), cachePath);
|
||||
await pruneDocumentPreviewCache(DOCUMENT_PREVIEW_CACHE_DIR);
|
||||
return cachePath;
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
|
||||
@@ -8,6 +8,7 @@ import { tmpdir } from 'node:os';
|
||||
import { basename, extname, join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { getOfficePreviewPdfPath } from './document-preview-cache.js';
|
||||
import { runWithConversionLimit } from './document-conversion-limiter.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const THUMBNAIL_CONVERSION_TIMEOUT_MS = 5 * 60_000;
|
||||
@@ -61,13 +62,11 @@ async function renderPdfFirstPage(filePath: string): Promise<ThumbnailResult | n
|
||||
try {
|
||||
previewDir = await fs.mkdtemp(join(tmpdir(), 'codeman-thumb-pdf-'));
|
||||
const prefix = join(previewDir, basename(filePath, extname(filePath)));
|
||||
await execFileAsync(
|
||||
'pdftoppm',
|
||||
['-png', '-singlefile', '-f', '1', '-l', '1', '-scale-to', '520', filePath, prefix],
|
||||
{
|
||||
await runWithConversionLimit(() =>
|
||||
execFileAsync('pdftoppm', ['-png', '-singlefile', '-f', '1', '-l', '1', '-scale-to', '520', filePath, prefix], {
|
||||
timeout: THUMBNAIL_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: 1024 * 1024,
|
||||
}
|
||||
})
|
||||
);
|
||||
const content = await fs.readFile(`${prefix}.png`);
|
||||
return { content, contentType: 'image/png' };
|
||||
|
||||
@@ -2509,6 +2509,23 @@ Object.assign(CodemanApp.prototype, {
|
||||
return;
|
||||
}
|
||||
|
||||
// Workspace-path (auto-detected, unregistered) attachments: Office docs are
|
||||
// converted to PDF server-side via the file-preview route; PDFs stream raw.
|
||||
// Both render inline in an iframe. Without this, docx/pptx/pdf fall through
|
||||
// to file-content below, which would dump the binary bytes as mojibake.
|
||||
if (ext === 'docx' || ext === 'pptx') {
|
||||
footerEl.textContent = ext.toUpperCase();
|
||||
const previewSrc = `/api/sessions/${sessionId}/file-preview?path=${encodeURIComponent(filePath)}`;
|
||||
bodyEl.innerHTML = `<iframe src="${escapeHtml(previewSrc)}" title="${escapeHtml(filePath)}"></iframe>`;
|
||||
return;
|
||||
}
|
||||
if (ext === 'pdf') {
|
||||
footerEl.textContent = 'PDF';
|
||||
const rawSrc = `/api/sessions/${sessionId}/file-raw?path=${encodeURIComponent(filePath)}`;
|
||||
bodyEl.innerHTML = `<iframe src="${escapeHtml(rawSrc)}" title="${escapeHtml(filePath)}"></iframe>`;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(`/api/sessions/${sessionId}/file-content?path=${encodeURIComponent(filePath)}&lines=500`);
|
||||
if (!res.ok) throw new Error('Failed to load file');
|
||||
|
||||
Reference in New Issue
Block a user