mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-08 16:39:42 +02:00
fix: Ralph wizard uses file-based prompt to avoid screen escaping issues
- Add /api/sessions/:id/ralph-prompt/write endpoint to write prompt to @ralph_prompt.md - Wizard now writes full prompt to file, then sends simple read command to Claude - Fix session readiness check to wait for prompt character instead of notWorking flag - Add E2E test infrastructure for ralph-loop workflow (port 3190) - Add ralph-wizard-prod.mjs script for production testing Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
arkon
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* Browser fixture for E2E tests
|
||||
* Manages Playwright browser lifecycle with required args for headless Chrome
|
||||
*/
|
||||
|
||||
import { chromium, Browser, BrowserContext, Page } from 'playwright';
|
||||
|
||||
export interface BrowserFixture {
|
||||
browser: Browser;
|
||||
context: BrowserContext;
|
||||
page: Page;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and launch a browser fixture
|
||||
* Uses required args for headless Chrome in CI/Linux environments
|
||||
* @returns BrowserFixture with browser, context, and page
|
||||
*/
|
||||
export async function createBrowserFixture(): Promise<BrowserFixture> {
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
'--disable-software-rasterizer',
|
||||
],
|
||||
});
|
||||
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
// Set default timeout for all operations
|
||||
page.setDefaultTimeout(15000);
|
||||
|
||||
return {
|
||||
browser,
|
||||
context,
|
||||
page,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Close and cleanup a browser fixture
|
||||
* @param fixture - Browser fixture to destroy
|
||||
*/
|
||||
export async function destroyBrowserFixture(fixture: BrowserFixture): Promise<void> {
|
||||
if (fixture.page) {
|
||||
await fixture.page.close().catch(() => {});
|
||||
}
|
||||
if (fixture.context) {
|
||||
await fixture.context.close().catch(() => {});
|
||||
}
|
||||
if (fixture.browser) {
|
||||
await fixture.browser.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a URL and wait for load
|
||||
* Note: Uses 'domcontentloaded' because SSE streams prevent 'networkidle'
|
||||
* @param page - Playwright page
|
||||
* @param url - URL to navigate to
|
||||
*/
|
||||
export async function navigateTo(page: Page, url: string): Promise<void> {
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded' });
|
||||
// Wait a bit for JS to initialize
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be visible
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
* @param timeout - Timeout in ms (default 15000)
|
||||
*/
|
||||
export async function waitForVisible(page: Page, selector: string, timeout = 15000): Promise<void> {
|
||||
await page.waitForSelector(selector, { state: 'visible', timeout });
|
||||
}
|
||||
|
||||
/**
|
||||
* Click an element with retry logic
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
*/
|
||||
export async function clickElement(page: Page, selector: string): Promise<void> {
|
||||
await page.click(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Type text into an input element
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
* @param text - Text to type
|
||||
*/
|
||||
export async function typeInto(page: Page, selector: string, text: string): Promise<void> {
|
||||
await page.fill(selector, text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get text content of an element
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
* @returns Text content or empty string
|
||||
*/
|
||||
export async function getText(page: Page, selector: string): Promise<string> {
|
||||
try {
|
||||
const element = await page.$(selector);
|
||||
if (element) {
|
||||
return (await element.textContent()) || '';
|
||||
}
|
||||
return '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an element is visible
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
* @returns true if visible, false otherwise
|
||||
*/
|
||||
export async function isVisible(page: Page, selector: string): Promise<boolean> {
|
||||
try {
|
||||
const element = await page.$(selector);
|
||||
if (element) {
|
||||
return await element.isVisible();
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of elements matching selector
|
||||
* @param page - Playwright page
|
||||
* @param selector - CSS selector
|
||||
* @returns Number of matching elements
|
||||
*/
|
||||
export async function getElementCount(page: Page, selector: string): Promise<number> {
|
||||
return await page.locator(selector).count();
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Cleanup fixture for E2E tests
|
||||
* Tracks and cleans up all resources (sessions, cases, screens)
|
||||
*/
|
||||
|
||||
import { execSync } from 'node:child_process';
|
||||
import { existsSync, rmSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export class CleanupTracker {
|
||||
private sessions: Set<string> = new Set();
|
||||
private cases: Set<string> = new Set();
|
||||
private screens: Set<string> = new Set();
|
||||
private baseUrl: string;
|
||||
private casesDir: string;
|
||||
|
||||
constructor(baseUrl: string) {
|
||||
this.baseUrl = baseUrl;
|
||||
this.casesDir = join(homedir(), 'claudeman-cases');
|
||||
}
|
||||
|
||||
/**
|
||||
* Track a session for cleanup
|
||||
*/
|
||||
trackSession(sessionId: string): void {
|
||||
this.sessions.add(sessionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Track a case for cleanup
|
||||
*/
|
||||
trackCase(caseName: string): void {
|
||||
this.cases.add(caseName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Track a screen for cleanup
|
||||
*/
|
||||
trackScreen(screenName: string): void {
|
||||
this.screens.add(screenName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all tracked resources
|
||||
*/
|
||||
async cleanup(): Promise<void> {
|
||||
// Delete sessions via API
|
||||
for (const sessionId of this.sessions) {
|
||||
try {
|
||||
await fetch(`${this.baseUrl}/api/sessions/${sessionId}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
} catch {
|
||||
// Ignore errors, session may already be deleted
|
||||
}
|
||||
}
|
||||
this.sessions.clear();
|
||||
|
||||
// Delete case directories
|
||||
for (const caseName of this.cases) {
|
||||
try {
|
||||
const casePath = join(this.casesDir, caseName);
|
||||
if (existsSync(casePath)) {
|
||||
rmSync(casePath, { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors
|
||||
}
|
||||
}
|
||||
this.cases.clear();
|
||||
|
||||
// Kill tracked screens
|
||||
for (const screenName of this.screens) {
|
||||
this.killScreen(screenName);
|
||||
}
|
||||
this.screens.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Kill a specific screen session
|
||||
*/
|
||||
private killScreen(screenName: string): void {
|
||||
try {
|
||||
execSync(`screen -S ${screenName} -X quit 2>/dev/null`, {
|
||||
stdio: 'pipe',
|
||||
timeout: 5000,
|
||||
});
|
||||
} catch {
|
||||
// Screen may not exist or already be dead
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force cleanup ALL e2e-test-* cases and claudeman-* screens
|
||||
* Use this in afterAll to ensure no orphans remain
|
||||
*/
|
||||
async forceCleanupAll(): Promise<void> {
|
||||
// First, clean tracked resources
|
||||
await this.cleanup();
|
||||
|
||||
// Force kill ALL e2e-test sessions via API
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/api/sessions`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
if (data.sessions) {
|
||||
for (const session of data.sessions) {
|
||||
if (session.name?.startsWith('e2e-test-') || session.workingDir?.includes('e2e-test-')) {
|
||||
try {
|
||||
await fetch(`${this.baseUrl}/api/sessions/${session.id}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Server may be down
|
||||
}
|
||||
|
||||
// Clean up e2e-test-* case directories
|
||||
try {
|
||||
if (existsSync(this.casesDir)) {
|
||||
const entries = execSync(`ls -1 "${this.casesDir}" 2>/dev/null || true`, {
|
||||
encoding: 'utf-8',
|
||||
timeout: 5000,
|
||||
}).trim().split('\n').filter(Boolean);
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.startsWith('e2e-test-')) {
|
||||
const casePath = join(this.casesDir, entry);
|
||||
rmSync(casePath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
|
||||
// Kill all e2e-test claudeman screens
|
||||
try {
|
||||
const screenList = execSync('screen -ls 2>/dev/null || true', {
|
||||
encoding: 'utf-8',
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
const screenLines = screenList.split('\n');
|
||||
for (const line of screenLines) {
|
||||
// Match claudeman-e2e-test-* screens
|
||||
const match = line.match(/\d+\.(claudeman-e2e-test-[^\s]+)/);
|
||||
if (match) {
|
||||
const screenName = match[1];
|
||||
this.killScreen(screenName);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore screen cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of screens that match a pattern
|
||||
*/
|
||||
getScreensByPattern(pattern: string): string[] {
|
||||
try {
|
||||
const screenList = execSync('screen -ls 2>/dev/null || true', {
|
||||
encoding: 'utf-8',
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
const screens: string[] = [];
|
||||
const screenLines = screenList.split('\n');
|
||||
for (const line of screenLines) {
|
||||
if (line.includes(pattern)) {
|
||||
const match = line.match(/\d+\.([^\s]+)/);
|
||||
if (match) {
|
||||
screens.push(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return screens;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a screen exists
|
||||
*/
|
||||
screenExists(screenName: string): boolean {
|
||||
try {
|
||||
const screenList = execSync('screen -ls 2>/dev/null || true', {
|
||||
encoding: 'utf-8',
|
||||
timeout: 5000,
|
||||
});
|
||||
return screenList.includes(screenName);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* E2E Test Fixtures
|
||||
* Central export for all fixture utilities
|
||||
*/
|
||||
|
||||
export {
|
||||
createServerFixture,
|
||||
destroyServerFixture,
|
||||
type ServerFixture,
|
||||
} from './server.fixture.js';
|
||||
|
||||
export {
|
||||
createBrowserFixture,
|
||||
destroyBrowserFixture,
|
||||
navigateTo,
|
||||
waitForVisible,
|
||||
clickElement,
|
||||
typeInto,
|
||||
getText,
|
||||
isVisible,
|
||||
getElementCount,
|
||||
type BrowserFixture,
|
||||
} from './browser.fixture.js';
|
||||
|
||||
export {
|
||||
CleanupTracker,
|
||||
} from './cleanup.fixture.js';
|
||||
|
||||
export {
|
||||
captureAndCompare,
|
||||
captureScreenshot,
|
||||
updateBaseline,
|
||||
type ScreenshotOptions,
|
||||
type ScreenshotResult,
|
||||
} from './screenshot.fixture.js';
|
||||
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Type declarations for pixelmatch
|
||||
*/
|
||||
declare module 'pixelmatch' {
|
||||
interface PixelmatchOptions {
|
||||
threshold?: number;
|
||||
includeAA?: boolean;
|
||||
alpha?: number;
|
||||
aaColor?: [number, number, number];
|
||||
diffColor?: [number, number, number];
|
||||
diffColorAlt?: [number, number, number] | null;
|
||||
diffMask?: boolean;
|
||||
}
|
||||
|
||||
function pixelmatch(
|
||||
img1: Uint8Array | Uint8ClampedArray | Buffer,
|
||||
img2: Uint8Array | Uint8ClampedArray | Buffer,
|
||||
output: Uint8Array | Uint8ClampedArray | Buffer | null,
|
||||
width: number,
|
||||
height: number,
|
||||
options?: PixelmatchOptions
|
||||
): number;
|
||||
|
||||
export = pixelmatch;
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* Screenshot fixture for E2E tests
|
||||
* Captures screenshots and compares against baselines for visual regression testing
|
||||
*/
|
||||
|
||||
import { Page } from 'playwright';
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { PNG } from 'pngjs';
|
||||
import pixelmatch from 'pixelmatch';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const SCREENSHOTS_DIR = join(__dirname, '..', 'screenshots');
|
||||
const BASELINES_DIR = join(SCREENSHOTS_DIR, 'baselines');
|
||||
const CURRENT_DIR = join(SCREENSHOTS_DIR, 'current');
|
||||
const DIFFS_DIR = join(SCREENSHOTS_DIR, 'diffs');
|
||||
|
||||
// Default threshold - percentage of pixels allowed to differ (0-1)
|
||||
const DEFAULT_THRESHOLD = 0.05; // 5%
|
||||
|
||||
export interface ScreenshotOptions {
|
||||
/** Threshold for pixel difference (0-1), default 0.05 */
|
||||
threshold?: number;
|
||||
/** Mask selectors to exclude from comparison */
|
||||
mask?: string[];
|
||||
/** Full page screenshot vs viewport only */
|
||||
fullPage?: boolean;
|
||||
}
|
||||
|
||||
export interface ScreenshotResult {
|
||||
/** Whether comparison passed */
|
||||
passed: boolean;
|
||||
/** Path to baseline image */
|
||||
baselinePath: string;
|
||||
/** Path to current image */
|
||||
currentPath: string;
|
||||
/** Path to diff image (only if comparison failed) */
|
||||
diffPath?: string;
|
||||
/** Number of different pixels */
|
||||
diffPixels?: number;
|
||||
/** Percentage of different pixels */
|
||||
diffPercent?: number;
|
||||
/** Whether this is a new baseline (no previous baseline existed) */
|
||||
newBaseline: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure screenshot directories exist
|
||||
*/
|
||||
function ensureDirectories(): void {
|
||||
for (const dir of [SCREENSHOTS_DIR, BASELINES_DIR, CURRENT_DIR, DIFFS_DIR]) {
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a screenshot and compare against baseline
|
||||
* @param page - Playwright page
|
||||
* @param name - Screenshot name (without extension)
|
||||
* @param options - Screenshot options
|
||||
* @returns ScreenshotResult with comparison details
|
||||
*/
|
||||
export async function captureAndCompare(
|
||||
page: Page,
|
||||
name: string,
|
||||
options: ScreenshotOptions = {}
|
||||
): Promise<ScreenshotResult> {
|
||||
ensureDirectories();
|
||||
|
||||
const threshold = options.threshold ?? DEFAULT_THRESHOLD;
|
||||
const baselinePath = join(BASELINES_DIR, `${name}.png`);
|
||||
const currentPath = join(CURRENT_DIR, `${name}.png`);
|
||||
const diffPath = join(DIFFS_DIR, `${name}.png`);
|
||||
|
||||
// Mask elements if specified (hide dynamic content)
|
||||
if (options.mask && options.mask.length > 0) {
|
||||
for (const selector of options.mask) {
|
||||
await page.evaluate((sel) => {
|
||||
const elements = document.querySelectorAll(sel);
|
||||
elements.forEach(el => {
|
||||
(el as HTMLElement).style.visibility = 'hidden';
|
||||
});
|
||||
}, selector);
|
||||
}
|
||||
}
|
||||
|
||||
// Capture current screenshot
|
||||
const screenshotBuffer = await page.screenshot({
|
||||
path: currentPath,
|
||||
fullPage: options.fullPage ?? false,
|
||||
});
|
||||
|
||||
// Restore masked elements
|
||||
if (options.mask && options.mask.length > 0) {
|
||||
for (const selector of options.mask) {
|
||||
await page.evaluate((sel) => {
|
||||
const elements = document.querySelectorAll(sel);
|
||||
elements.forEach(el => {
|
||||
(el as HTMLElement).style.visibility = 'visible';
|
||||
});
|
||||
}, selector);
|
||||
}
|
||||
}
|
||||
|
||||
// If no baseline exists, create one
|
||||
if (!existsSync(baselinePath)) {
|
||||
writeFileSync(baselinePath, screenshotBuffer);
|
||||
return {
|
||||
passed: true,
|
||||
baselinePath,
|
||||
currentPath,
|
||||
newBaseline: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Load baseline and current images
|
||||
const baselineBuffer = readFileSync(baselinePath);
|
||||
const baselineImg = PNG.sync.read(baselineBuffer);
|
||||
const currentImg = PNG.sync.read(screenshotBuffer);
|
||||
|
||||
// Check dimensions match
|
||||
if (baselineImg.width !== currentImg.width || baselineImg.height !== currentImg.height) {
|
||||
// Dimensions changed - save diff and fail
|
||||
return {
|
||||
passed: false,
|
||||
baselinePath,
|
||||
currentPath,
|
||||
diffPath,
|
||||
diffPixels: baselineImg.width * baselineImg.height,
|
||||
diffPercent: 1,
|
||||
newBaseline: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Compare images
|
||||
const { width, height } = baselineImg;
|
||||
const diffImg = new PNG({ width, height });
|
||||
const diffPixels = pixelmatch(
|
||||
baselineImg.data,
|
||||
currentImg.data,
|
||||
diffImg.data,
|
||||
width,
|
||||
height,
|
||||
{ threshold: 0.1 } // pixelmatch threshold (per-pixel sensitivity)
|
||||
);
|
||||
|
||||
const totalPixels = width * height;
|
||||
const diffPercent = diffPixels / totalPixels;
|
||||
const passed = diffPercent <= threshold;
|
||||
|
||||
// Save diff image if comparison failed
|
||||
if (!passed) {
|
||||
writeFileSync(diffPath, PNG.sync.write(diffImg));
|
||||
}
|
||||
|
||||
return {
|
||||
passed,
|
||||
baselinePath,
|
||||
currentPath,
|
||||
diffPath: passed ? undefined : diffPath,
|
||||
diffPixels,
|
||||
diffPercent,
|
||||
newBaseline: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a screenshot without comparison (for debugging)
|
||||
* @param page - Playwright page
|
||||
* @param name - Screenshot name
|
||||
*/
|
||||
export async function captureScreenshot(page: Page, name: string): Promise<string> {
|
||||
ensureDirectories();
|
||||
const path = join(CURRENT_DIR, `${name}.png`);
|
||||
await page.screenshot({ path });
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update baseline with current screenshot
|
||||
* @param name - Screenshot name
|
||||
*/
|
||||
export function updateBaseline(name: string): boolean {
|
||||
const currentPath = join(CURRENT_DIR, `${name}.png`);
|
||||
const baselinePath = join(BASELINES_DIR, `${name}.png`);
|
||||
|
||||
if (!existsSync(currentPath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentBuffer = readFileSync(currentPath);
|
||||
writeFileSync(baselinePath, currentBuffer);
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Server fixture for E2E tests
|
||||
* Manages WebServer lifecycle with proper startup/shutdown
|
||||
*/
|
||||
|
||||
import { WebServer } from '../../../src/web/server.js';
|
||||
|
||||
export interface ServerFixture {
|
||||
server: WebServer;
|
||||
port: number;
|
||||
baseUrl: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and start a server fixture
|
||||
* @param port - Port to run the server on
|
||||
* @returns ServerFixture with server instance and connection info
|
||||
*/
|
||||
export async function createServerFixture(port: number): Promise<ServerFixture> {
|
||||
const server = new WebServer(port);
|
||||
await server.start();
|
||||
|
||||
// Wait for server to be fully ready
|
||||
const maxWait = 10000;
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < maxWait) {
|
||||
try {
|
||||
const response = await fetch(`http://localhost:${port}/api/status`);
|
||||
if (response.ok) {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Server not ready yet
|
||||
}
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
}
|
||||
|
||||
return {
|
||||
server,
|
||||
port,
|
||||
baseUrl: `http://localhost:${port}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop and cleanup a server fixture
|
||||
* @param fixture - Server fixture to destroy
|
||||
*/
|
||||
export async function destroyServerFixture(fixture: ServerFixture): Promise<void> {
|
||||
if (fixture.server) {
|
||||
await fixture.server.stop();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user