mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
fix: subagent tabs render to correct parent session after restart
- restoreSubagentWindowStates now discovers parent sessions before restoring - closeSubagentWindow discovers parent before minimizing to prevent wrong tab - Fixed async handling in subagent:completed event handler chore: bump version to 0.1389 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
arkon
@@ -0,0 +1,487 @@
|
||||
/**
|
||||
* Agent-Browser Enhanced Test Suite
|
||||
*
|
||||
* Tests Claudeman web UI using agent-browser's AI-agent-friendly features:
|
||||
* - Semantic locators (find by role, label, text)
|
||||
* - Accessibility tree snapshots
|
||||
* - Reference-based element selection (@e1, @e2)
|
||||
*
|
||||
* This demonstrates the "agentic" approach to browser testing where an AI
|
||||
* could navigate the UI using natural language descriptions.
|
||||
*
|
||||
* Port: 3157 (see CLAUDE.md test port table)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import { execSync, spawnSync } from 'node:child_process';
|
||||
import { WebServer } from '../src/web/server.js';
|
||||
|
||||
const TEST_PORT = 3157;
|
||||
const BASE_URL = `http://localhost:${TEST_PORT}`;
|
||||
const BROWSER_TIMEOUT = 30000;
|
||||
|
||||
// Metrics collection
|
||||
interface TestMetrics {
|
||||
testName: string;
|
||||
duration: number;
|
||||
framework: 'agent-browser';
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const metrics: TestMetrics[] = [];
|
||||
|
||||
// ============================================================================
|
||||
// Agent-Browser CLI Helpers
|
||||
// ============================================================================
|
||||
|
||||
function agentBrowser(command: string): string {
|
||||
try {
|
||||
return execSync(`npx agent-browser ${command}`, {
|
||||
timeout: BROWSER_TIMEOUT,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
}).trim();
|
||||
} catch (error: any) {
|
||||
if (error.stderr) {
|
||||
throw new Error(`agent-browser failed: ${error.stderr}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function agentBrowserJson<T = any>(command: string): T {
|
||||
const result = agentBrowser(`${command} --json`);
|
||||
const parsed = JSON.parse(result);
|
||||
if (!parsed.success) {
|
||||
throw new Error(`agent-browser command failed: ${parsed.error || 'unknown error'}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
// Get accessibility tree snapshot for AI-based navigation
|
||||
function getSnapshot(): string {
|
||||
try {
|
||||
return agentBrowser('snapshot');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Find element by semantic criteria (AI-friendly)
|
||||
function findByRole(role: string, name?: string): boolean {
|
||||
try {
|
||||
const cmd = name
|
||||
? `find role ${role} --name "${name}"`
|
||||
: `find role ${role}`;
|
||||
const result = agentBrowser(cmd);
|
||||
return result.includes('Found') || !result.includes('not found');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findByText(text: string): boolean {
|
||||
try {
|
||||
const result = agentBrowser(`find text "${text}"`);
|
||||
return result.includes('Found') || !result.includes('not found');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findByLabel(label: string): boolean {
|
||||
try {
|
||||
const result = agentBrowser(`find label "${label}"`);
|
||||
return result.includes('Found') || !result.includes('not found');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Click using semantic finder
|
||||
function clickByText(text: string): boolean {
|
||||
try {
|
||||
agentBrowser(`find text "${text}" click`);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function clickByRole(role: string, name?: string): boolean {
|
||||
try {
|
||||
const cmd = name
|
||||
? `find role ${role} click --name "${name}"`
|
||||
: `find role ${role} click`;
|
||||
agentBrowser(cmd);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Standard helpers
|
||||
function getText(selector: string): string {
|
||||
try {
|
||||
return agentBrowserJson<{ text: string }>(`get text "${selector}"`).text || '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function isVisible(selector: string): boolean {
|
||||
try {
|
||||
return agentBrowserJson<{ visible: boolean }>(`is visible "${selector}"`).visible;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function getCount(selector: string): number {
|
||||
try {
|
||||
return agentBrowserJson<{ count: number }>(`get count "${selector}"`).count;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForElement(selector: string, timeout = 10000): Promise<boolean> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeout) {
|
||||
if (getCount(selector) > 0) return true;
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function closeBrowser() {
|
||||
try {
|
||||
agentBrowser('close');
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
describe('Agent-Browser Enhanced Tests', () => {
|
||||
let server: WebServer;
|
||||
let createdSessions: string[] = [];
|
||||
let browserAvailable = false;
|
||||
let testStartTime: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
closeBrowser();
|
||||
|
||||
// Start web server
|
||||
server = new WebServer(TEST_PORT);
|
||||
await server.start();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Test if agent-browser is available
|
||||
try {
|
||||
agentBrowser(`open ${BASE_URL}`);
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
const title = agentBrowserJson<{ title: string }>('get title');
|
||||
browserAvailable = title.title === 'Claudeman';
|
||||
} catch (e) {
|
||||
console.warn('Agent-browser not available, skipping tests:', (e as Error).message);
|
||||
browserAvailable = false;
|
||||
}
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
closeBrowser();
|
||||
|
||||
// Cleanup sessions
|
||||
for (const sessionId of createdSessions) {
|
||||
try {
|
||||
await fetch(`${BASE_URL}/api/sessions/${sessionId}`, { method: 'DELETE' });
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
await server.stop();
|
||||
|
||||
// Output metrics
|
||||
console.log('\n📊 Agent-Browser Test Metrics:');
|
||||
console.table(metrics.map(m => ({
|
||||
test: m.testName.substring(0, 40),
|
||||
duration: `${m.duration}ms`,
|
||||
status: m.success ? '✅' : '❌',
|
||||
})));
|
||||
}, 60000);
|
||||
|
||||
beforeEach(() => {
|
||||
testStartTime = Date.now();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
const testInfo = expect.getState();
|
||||
metrics.push({
|
||||
testName: testInfo.currentTestName || 'unknown',
|
||||
duration: Date.now() - testStartTime,
|
||||
framework: 'agent-browser',
|
||||
success: !testInfo.assertionCalls?.some((c: any) => !c.pass),
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Core Page Tests (CSS Selectors)
|
||||
// ============================================================================
|
||||
|
||||
it('should load the main page with correct title', async () => {
|
||||
if (!browserAvailable) {
|
||||
console.log('Skipping: browser not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const title = agentBrowserJson<{ title: string }>('get title');
|
||||
expect(title.title).toBe('Claudeman');
|
||||
});
|
||||
|
||||
it('should display the logo and branding', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
const logoText = getText('.header-brand .logo');
|
||||
expect(logoText).toBe('Claudeman');
|
||||
});
|
||||
|
||||
it('should show version in toolbar', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
const version = getText('#versionDisplay');
|
||||
expect(version).toMatch(/v\d+\.\d+/);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Semantic/AI-Friendly Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should find elements using semantic role locators', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Find buttons by role
|
||||
const hasButtons = findByRole('button');
|
||||
expect(hasButtons).toBe(true);
|
||||
|
||||
// Find specific buttons (may vary based on page structure)
|
||||
// agent-browser uses ARIA roles for semantic finding
|
||||
});
|
||||
|
||||
it('should capture accessibility tree snapshot', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
const snapshot = getSnapshot();
|
||||
|
||||
// Snapshot should contain accessibility information
|
||||
expect(snapshot.length).toBeGreaterThan(0);
|
||||
|
||||
// Should include key page elements
|
||||
// The snapshot format varies by agent-browser version
|
||||
console.log('Accessibility snapshot captured, length:', snapshot.length);
|
||||
});
|
||||
|
||||
it('should find elements by text content', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Find elements containing specific text
|
||||
const foundClaudeman = findByText('Claudeman');
|
||||
expect(foundClaudeman).toBe(true);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Modal Tests (CSS + Semantic)
|
||||
// ============================================================================
|
||||
|
||||
it('should open and close help modal', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Open using CSS selector
|
||||
agentBrowser('click ".help-btn"');
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
expect(isVisible('#helpModal .modal-content')).toBe(true);
|
||||
|
||||
// Find heading by text
|
||||
const headingText = getText('#helpModal h3');
|
||||
expect(headingText).toContain('Keyboard Shortcuts');
|
||||
|
||||
// Close modal
|
||||
agentBrowser('click "#helpModal .modal-close"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
expect(isVisible('#helpModal .modal-content')).toBe(false);
|
||||
});
|
||||
|
||||
it('should open settings modal with correct tabs', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
agentBrowser('click ".btn-settings"');
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
expect(isVisible('#appSettingsModal .modal-content')).toBe(true);
|
||||
|
||||
const tabCount = getCount('#appSettingsModal .modal-tab-btn');
|
||||
expect(tabCount).toBe(4);
|
||||
|
||||
agentBrowser('click "#appSettingsModal .modal-close"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Session Management Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should create a new session via button click', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Click create session
|
||||
agentBrowser('click ".btn-claude"');
|
||||
|
||||
// Wait for session tab
|
||||
const tabFound = await waitForElement('.session-tab', 15000);
|
||||
expect(tabFound).toBe(true);
|
||||
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
expect(isVisible('.session-tab.active')).toBe(true);
|
||||
|
||||
// Terminal should be visible
|
||||
const xtermVisible = await waitForElement('.xterm', 10000);
|
||||
expect(xtermVisible).toBe(true);
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`${BASE_URL}/api/sessions`);
|
||||
const data = await response.json();
|
||||
if (data.sessions?.length > 0) {
|
||||
const lastSession = data.sessions[data.sessions.length - 1];
|
||||
if (lastSession && !createdSessions.includes(lastSession.id)) {
|
||||
createdSessions.push(lastSession.id);
|
||||
}
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// AI-Driven Navigation Demo
|
||||
// ============================================================================
|
||||
|
||||
it('should demonstrate AI-friendly navigation workflow', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// This test demonstrates how an AI agent could navigate the UI
|
||||
// using semantic locators and natural language-like commands
|
||||
|
||||
// Step 1: Check page state via snapshot
|
||||
const snapshot = getSnapshot();
|
||||
expect(snapshot.length).toBeGreaterThan(0);
|
||||
|
||||
// Step 2: Find and interact with settings
|
||||
// An AI would parse the snapshot and decide what to click
|
||||
const settingsVisible = isVisible('.btn-settings');
|
||||
expect(settingsVisible).toBe(true);
|
||||
|
||||
// Step 3: Open settings
|
||||
agentBrowser('click ".btn-settings"');
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
// Step 4: Take new snapshot to understand modal state
|
||||
const modalSnapshot = getSnapshot();
|
||||
|
||||
// Step 5: Close settings
|
||||
agentBrowser('click "#appSettingsModal .modal-close"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
// This workflow shows how an AI could:
|
||||
// 1. Observe the page state (snapshot)
|
||||
// 2. Decide on actions based on semantic understanding
|
||||
// 3. Execute actions
|
||||
// 4. Verify results with new snapshot
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Form Interaction Tests (Semantic)
|
||||
// ============================================================================
|
||||
|
||||
it('should interact with quick start dropdown', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Find the quick start dropdown
|
||||
const dropdownVisible = isVisible('#quickStartCase');
|
||||
expect(dropdownVisible).toBe(true);
|
||||
|
||||
// Get current value
|
||||
// Note: agent-browser's get value command for select elements
|
||||
try {
|
||||
const value = agentBrowserJson<{ value: string }>('get value "#quickStartCase"');
|
||||
// Value should be the current selected case
|
||||
expect(typeof value.value).toBe('string');
|
||||
} catch {
|
||||
// May not be a standard select element
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Multiple Element Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should count toolbar elements correctly', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Count various toolbar elements
|
||||
const fontControls = getCount('.header-font-controls .font-btn');
|
||||
expect(fontControls).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Tab count badge should exist
|
||||
expect(isVisible('#tabCount')).toBe(true);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Screenshot Test
|
||||
// ============================================================================
|
||||
|
||||
it('should capture screenshot for AI analysis', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
try {
|
||||
// Capture screenshot (useful for visual AI analysis)
|
||||
agentBrowser('screenshot /tmp/claudeman-test.png');
|
||||
|
||||
// Verify file was created (would exist after successful screenshot)
|
||||
// In a real AI scenario, this image would be analyzed
|
||||
} catch {
|
||||
// Screenshot may fail in headless mode without proper setup
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Stress Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should handle rapid interactions', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
// Rapid modal open/close
|
||||
for (let i = 0; i < 3; i++) {
|
||||
agentBrowser('click ".help-btn"');
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
try {
|
||||
agentBrowser('click "#helpModal .modal-close"');
|
||||
} catch { /* may already be closed */ }
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
}
|
||||
|
||||
// Page should still be functional
|
||||
expect(isVisible('.header')).toBe(true);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Tab Navigation Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should handle tab count display', async () => {
|
||||
if (!browserAvailable) return;
|
||||
|
||||
const tabCountText = getText('#tabCount');
|
||||
// Should show number of tabs
|
||||
expect(tabCountText).toMatch(/\d+/);
|
||||
});
|
||||
});
|
||||
|
||||
// Export metrics for comparison
|
||||
export { metrics as agentBrowserMetrics };
|
||||
@@ -0,0 +1,624 @@
|
||||
/**
|
||||
* Browser Framework Comparison Benchmark
|
||||
*
|
||||
* Runs identical test scenarios across Playwright, Puppeteer, and Agent-Browser
|
||||
* to compare performance, reliability, and ease of use for agentic testing.
|
||||
*
|
||||
* This benchmark measures:
|
||||
* - Execution time per operation
|
||||
* - Reliability (success rate)
|
||||
* - Memory usage (where available)
|
||||
* - Ease of implementation (qualitative)
|
||||
*
|
||||
* Ports: 3158 (Playwright), 3159 (Puppeteer), 3160 (Agent-Browser)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { chromium, Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright';
|
||||
import puppeteer, { Browser as PuppeteerBrowser, Page as PuppeteerPage } from 'puppeteer';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { WebServer } from '../src/web/server.js';
|
||||
|
||||
// Port allocation
|
||||
const PORTS = {
|
||||
playwright: 3158,
|
||||
puppeteer: 3159,
|
||||
agentBrowser: 3160,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
interface BenchmarkResult {
|
||||
scenario: string;
|
||||
framework: 'playwright' | 'puppeteer' | 'agent-browser';
|
||||
duration: number;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface AggregateStats {
|
||||
framework: string;
|
||||
totalTests: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
avgDuration: number;
|
||||
minDuration: number;
|
||||
maxDuration: number;
|
||||
successRate: string;
|
||||
}
|
||||
|
||||
const results: BenchmarkResult[] = [];
|
||||
|
||||
// ============================================================================
|
||||
// Agent-Browser Helpers
|
||||
// ============================================================================
|
||||
|
||||
function agentBrowser(command: string): string {
|
||||
try {
|
||||
return execSync(`npx agent-browser ${command}`, {
|
||||
timeout: 30000,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
}).trim();
|
||||
} catch (error: any) {
|
||||
throw new Error(`agent-browser failed: ${error.stderr || error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function agentBrowserJson<T>(command: string): T {
|
||||
const result = agentBrowser(`${command} --json`);
|
||||
const parsed = JSON.parse(result);
|
||||
if (!parsed.success) {
|
||||
throw new Error(`agent-browser: ${parsed.error || 'unknown error'}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Benchmark Scenarios
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Each scenario is a function that takes framework-specific parameters
|
||||
* and returns a promise that resolves when the scenario is complete.
|
||||
*/
|
||||
|
||||
async function runBenchmark(
|
||||
scenario: string,
|
||||
framework: 'playwright' | 'puppeteer' | 'agent-browser',
|
||||
fn: () => Promise<void>
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
let success = false;
|
||||
let error: string | undefined;
|
||||
|
||||
try {
|
||||
await fn();
|
||||
success = true;
|
||||
} catch (e: any) {
|
||||
error = e.message;
|
||||
}
|
||||
|
||||
results.push({
|
||||
scenario,
|
||||
framework,
|
||||
duration: Date.now() - start,
|
||||
success,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main Comparison Suite
|
||||
// ============================================================================
|
||||
|
||||
describe('Browser Framework Comparison Benchmark', () => {
|
||||
// Framework instances
|
||||
let playwrightBrowser: PlaywrightBrowser;
|
||||
let puppeteerBrowser: PuppeteerBrowser;
|
||||
let agentBrowserAvailable = false;
|
||||
|
||||
// Web servers
|
||||
let servers: WebServer[] = [];
|
||||
|
||||
// Cleanup tracking
|
||||
let createdSessions: { port: number; id: string }[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
console.log('\n🚀 Starting Browser Framework Comparison Benchmark\n');
|
||||
console.log('This benchmark compares:');
|
||||
console.log(' - Playwright (direct API)');
|
||||
console.log(' - Puppeteer (Chrome DevTools Protocol)');
|
||||
console.log(' - Agent-Browser (CLI-based, AI-friendly)\n');
|
||||
|
||||
// Start web servers for each framework
|
||||
for (const [name, port] of Object.entries(PORTS)) {
|
||||
const server = new WebServer(port);
|
||||
await server.start();
|
||||
servers.push(server);
|
||||
console.log(`✅ Started server for ${name} on port ${port}`);
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Initialize Playwright
|
||||
try {
|
||||
playwrightBrowser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
console.log('✅ Playwright browser launched');
|
||||
} catch (e: any) {
|
||||
console.warn('⚠️ Playwright launch failed:', e.message);
|
||||
}
|
||||
|
||||
// Initialize Puppeteer
|
||||
try {
|
||||
puppeteerBrowser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
|
||||
});
|
||||
console.log('✅ Puppeteer browser launched');
|
||||
} catch (e: any) {
|
||||
console.warn('⚠️ Puppeteer launch failed:', e.message);
|
||||
}
|
||||
|
||||
// Test Agent-Browser availability
|
||||
try {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
const title = agentBrowserJson<{ title: string }>('get title');
|
||||
agentBrowserAvailable = title.title === 'Claudeman';
|
||||
if (agentBrowserAvailable) {
|
||||
console.log('✅ Agent-Browser available');
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.warn('⚠️ Agent-Browser not available:', e.message);
|
||||
}
|
||||
|
||||
console.log('\n' + '='.repeat(60) + '\n');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
// Cleanup sessions
|
||||
for (const { port, id } of createdSessions) {
|
||||
try {
|
||||
await fetch(`http://localhost:${port}/api/sessions/${id}`, { method: 'DELETE' });
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Close browsers
|
||||
if (playwrightBrowser) await playwrightBrowser.close();
|
||||
if (puppeteerBrowser) await puppeteerBrowser.close();
|
||||
try { agentBrowser('close'); } catch { /* ignore */ }
|
||||
|
||||
// Stop servers
|
||||
for (const server of servers) {
|
||||
await server.stop();
|
||||
}
|
||||
|
||||
// Print results summary
|
||||
printResultsSummary();
|
||||
}, 60000);
|
||||
|
||||
// ============================================================================
|
||||
// Scenario 1: Page Load
|
||||
// ============================================================================
|
||||
|
||||
describe('Scenario: Page Load', () => {
|
||||
it('Playwright - page load', async () => {
|
||||
if (!playwrightBrowser) return;
|
||||
|
||||
await runBenchmark('page-load', 'playwright', async () => {
|
||||
const context = await playwrightBrowser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
await expect(page).toHaveTitle('Claudeman');
|
||||
await context.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Puppeteer - page load', async () => {
|
||||
if (!puppeteerBrowser) return;
|
||||
|
||||
await runBenchmark('page-load', 'puppeteer', async () => {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
const title = await page.title();
|
||||
expect(title).toBe('Claudeman');
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Agent-Browser - page load', async () => {
|
||||
if (!agentBrowserAvailable) return;
|
||||
|
||||
await runBenchmark('page-load', 'agent-browser', async () => {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
const title = agentBrowserJson<{ title: string }>('get title');
|
||||
expect(title.title).toBe('Claudeman');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Scenario 2: Element Selection
|
||||
// ============================================================================
|
||||
|
||||
describe('Scenario: Element Selection', () => {
|
||||
it('Playwright - element selection', async () => {
|
||||
if (!playwrightBrowser) return;
|
||||
|
||||
await runBenchmark('element-selection', 'playwright', async () => {
|
||||
const context = await playwrightBrowser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
|
||||
// Multiple element selections
|
||||
const logo = await page.locator('.header-brand .logo').textContent();
|
||||
const version = await page.locator('#versionDisplay').textContent();
|
||||
const btnCount = await page.locator('.btn-claude').count();
|
||||
|
||||
expect(logo).toBe('Claudeman');
|
||||
expect(version).toMatch(/v\d+/);
|
||||
expect(btnCount).toBeGreaterThan(0);
|
||||
|
||||
await context.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Puppeteer - element selection', async () => {
|
||||
if (!puppeteerBrowser) return;
|
||||
|
||||
await runBenchmark('element-selection', 'puppeteer', async () => {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
|
||||
const logo = await page.$eval('.header-brand .logo', el => el.textContent);
|
||||
const version = await page.$eval('#versionDisplay', el => el.textContent);
|
||||
const btns = await page.$$('.btn-claude');
|
||||
|
||||
expect(logo).toBe('Claudeman');
|
||||
expect(version).toMatch(/v\d+/);
|
||||
expect(btns.length).toBeGreaterThan(0);
|
||||
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Agent-Browser - element selection', async () => {
|
||||
if (!agentBrowserAvailable) return;
|
||||
|
||||
await runBenchmark('element-selection', 'agent-browser', async () => {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
const logo = agentBrowserJson<{ text: string }>('get text ".header-brand .logo"');
|
||||
const version = agentBrowserJson<{ text: string }>('get text "#versionDisplay"');
|
||||
const btnCount = agentBrowserJson<{ count: number }>('get count ".btn-claude"');
|
||||
|
||||
expect(logo.text).toBe('Claudeman');
|
||||
expect(version.text).toMatch(/v\d+/);
|
||||
expect(btnCount.count).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Scenario 3: Modal Interaction
|
||||
// ============================================================================
|
||||
|
||||
describe('Scenario: Modal Open/Close', () => {
|
||||
it('Playwright - modal interaction', async () => {
|
||||
if (!playwrightBrowser) return;
|
||||
|
||||
await runBenchmark('modal-interaction', 'playwright', async () => {
|
||||
const context = await playwrightBrowser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
|
||||
// Open modal
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'visible' });
|
||||
|
||||
// Verify content
|
||||
const heading = await page.locator('#helpModal h3').textContent();
|
||||
expect(heading).toContain('Keyboard Shortcuts');
|
||||
|
||||
// Close modal
|
||||
await page.click('#helpModal .modal-close');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'hidden' });
|
||||
|
||||
await context.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Puppeteer - modal interaction', async () => {
|
||||
if (!puppeteerBrowser) return;
|
||||
|
||||
await runBenchmark('modal-interaction', 'puppeteer', async () => {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
|
||||
// Open modal
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { visible: true });
|
||||
|
||||
// Verify content
|
||||
const heading = await page.$eval('#helpModal h3', el => el.textContent);
|
||||
expect(heading).toContain('Keyboard Shortcuts');
|
||||
|
||||
// Close modal
|
||||
await page.click('#helpModal .modal-close');
|
||||
await page.waitForFunction(() => {
|
||||
const modal = document.querySelector('#helpModal .modal-content');
|
||||
return !modal || window.getComputedStyle(modal).display === 'none';
|
||||
});
|
||||
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Agent-Browser - modal interaction', async () => {
|
||||
if (!agentBrowserAvailable) return;
|
||||
|
||||
await runBenchmark('modal-interaction', 'agent-browser', async () => {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Open modal
|
||||
agentBrowser('click ".help-btn"');
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
// Verify content
|
||||
const heading = agentBrowserJson<{ text: string }>('get text "#helpModal h3"');
|
||||
expect(heading.text).toContain('Keyboard Shortcuts');
|
||||
|
||||
// Close modal
|
||||
agentBrowser('click "#helpModal .modal-close"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Scenario 4: Session Creation (Complex)
|
||||
// ============================================================================
|
||||
|
||||
describe('Scenario: Session Creation', () => {
|
||||
it('Playwright - session creation', async () => {
|
||||
if (!playwrightBrowser) return;
|
||||
|
||||
await runBenchmark('session-creation', 'playwright', async () => {
|
||||
const context = await playwrightBrowser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
|
||||
// Create session
|
||||
await page.click('.btn-claude');
|
||||
|
||||
// Wait for session tab
|
||||
await page.waitForSelector('.session-tab', { timeout: 15000 });
|
||||
|
||||
// Verify terminal appears
|
||||
await page.waitForSelector('.xterm', { timeout: 10000 });
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`http://localhost:${PORTS.playwright}/api/sessions`);
|
||||
const data = await response.json();
|
||||
const lastSession = data.sessions?.[data.sessions.length - 1];
|
||||
if (lastSession) {
|
||||
createdSessions.push({ port: PORTS.playwright, id: lastSession.id });
|
||||
}
|
||||
|
||||
await context.close();
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it('Puppeteer - session creation', async () => {
|
||||
if (!puppeteerBrowser) return;
|
||||
|
||||
await runBenchmark('session-creation', 'puppeteer', async () => {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
|
||||
// Create session
|
||||
await page.click('.btn-claude');
|
||||
|
||||
// Wait for session tab
|
||||
await page.waitForSelector('.session-tab', { timeout: 15000 });
|
||||
|
||||
// Verify terminal appears
|
||||
await page.waitForSelector('.xterm', { timeout: 10000 });
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`http://localhost:${PORTS.puppeteer}/api/sessions`);
|
||||
const data = await response.json();
|
||||
const lastSession = data.sessions?.[data.sessions.length - 1];
|
||||
if (lastSession) {
|
||||
createdSessions.push({ port: PORTS.puppeteer, id: lastSession.id });
|
||||
}
|
||||
|
||||
await page.close();
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it('Agent-Browser - session creation', async () => {
|
||||
if (!agentBrowserAvailable) return;
|
||||
|
||||
await runBenchmark('session-creation', 'agent-browser', async () => {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Create session
|
||||
agentBrowser('click ".btn-claude"');
|
||||
|
||||
// Wait for session tab
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < 15000) {
|
||||
try {
|
||||
const count = agentBrowserJson<{ count: number }>('get count ".session-tab"');
|
||||
if (count.count > 0) break;
|
||||
} catch { /* retry */ }
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
}
|
||||
|
||||
// Verify terminal appears
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
const xtermCount = agentBrowserJson<{ count: number }>('get count ".xterm"');
|
||||
expect(xtermCount.count).toBeGreaterThan(0);
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`http://localhost:${PORTS.agentBrowser}/api/sessions`);
|
||||
const data = await response.json();
|
||||
const lastSession = data.sessions?.[data.sessions.length - 1];
|
||||
if (lastSession) {
|
||||
createdSessions.push({ port: PORTS.agentBrowser, id: lastSession.id });
|
||||
}
|
||||
});
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Scenario 5: Rapid Operations
|
||||
// ============================================================================
|
||||
|
||||
describe('Scenario: Rapid Operations (5 modal cycles)', () => {
|
||||
it('Playwright - rapid operations', async () => {
|
||||
if (!playwrightBrowser) return;
|
||||
|
||||
await runBenchmark('rapid-operations', 'playwright', async () => {
|
||||
const context = await playwrightBrowser.newContext();
|
||||
const page = await context.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'visible' });
|
||||
await page.click('#helpModal .modal-close');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'hidden' });
|
||||
}
|
||||
|
||||
await context.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Puppeteer - rapid operations', async () => {
|
||||
if (!puppeteerBrowser) return;
|
||||
|
||||
await runBenchmark('rapid-operations', 'puppeteer', async () => {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { visible: true });
|
||||
await page.click('#helpModal .modal-close');
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
}
|
||||
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
it('Agent-Browser - rapid operations', async () => {
|
||||
if (!agentBrowserAvailable) return;
|
||||
|
||||
await runBenchmark('rapid-operations', 'agent-browser', async () => {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
agentBrowser('click ".help-btn"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
agentBrowser('click "#helpModal .modal-close"');
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Results Summary
|
||||
// ============================================================================
|
||||
|
||||
function printResultsSummary(): void {
|
||||
console.log('\n' + '='.repeat(70));
|
||||
console.log('📊 BROWSER FRAMEWORK COMPARISON RESULTS');
|
||||
console.log('='.repeat(70) + '\n');
|
||||
|
||||
// Group by framework
|
||||
const frameworks = ['playwright', 'puppeteer', 'agent-browser'] as const;
|
||||
const stats: AggregateStats[] = [];
|
||||
|
||||
for (const framework of frameworks) {
|
||||
const frameworkResults = results.filter(r => r.framework === framework);
|
||||
if (frameworkResults.length === 0) continue;
|
||||
|
||||
const passed = frameworkResults.filter(r => r.success).length;
|
||||
const durations = frameworkResults.map(r => r.duration);
|
||||
|
||||
stats.push({
|
||||
framework,
|
||||
totalTests: frameworkResults.length,
|
||||
passed,
|
||||
failed: frameworkResults.length - passed,
|
||||
avgDuration: Math.round(durations.reduce((a, b) => a + b, 0) / durations.length),
|
||||
minDuration: Math.min(...durations),
|
||||
maxDuration: Math.max(...durations),
|
||||
successRate: ((passed / frameworkResults.length) * 100).toFixed(1) + '%',
|
||||
});
|
||||
}
|
||||
|
||||
// Print aggregate stats
|
||||
console.log('📈 AGGREGATE STATISTICS\n');
|
||||
console.table(stats.map(s => ({
|
||||
Framework: s.framework.toUpperCase(),
|
||||
Tests: s.totalTests,
|
||||
Passed: s.passed,
|
||||
Failed: s.failed,
|
||||
'Success Rate': s.successRate,
|
||||
'Avg (ms)': s.avgDuration,
|
||||
'Min (ms)': s.minDuration,
|
||||
'Max (ms)': s.maxDuration,
|
||||
})));
|
||||
|
||||
// Print detailed results by scenario
|
||||
console.log('\n📋 DETAILED RESULTS BY SCENARIO\n');
|
||||
|
||||
const scenarios = [...new Set(results.map(r => r.scenario))];
|
||||
for (const scenario of scenarios) {
|
||||
console.log(`\n🔹 ${scenario.toUpperCase()}`);
|
||||
const scenarioResults = results.filter(r => r.scenario === scenario);
|
||||
|
||||
console.table(scenarioResults.map(r => ({
|
||||
Framework: r.framework,
|
||||
Duration: `${r.duration}ms`,
|
||||
Status: r.success ? '✅ Pass' : '❌ Fail',
|
||||
Error: r.error ? r.error.substring(0, 40) + '...' : '-',
|
||||
})));
|
||||
}
|
||||
|
||||
// Performance comparison
|
||||
console.log('\n🏆 PERFORMANCE RANKING (by avg duration)\n');
|
||||
const ranked = [...stats].sort((a, b) => a.avgDuration - b.avgDuration);
|
||||
ranked.forEach((s, i) => {
|
||||
const medal = i === 0 ? '🥇' : i === 1 ? '🥈' : '🥉';
|
||||
console.log(`${medal} ${s.framework.toUpperCase()}: ${s.avgDuration}ms avg (${s.successRate} success rate)`);
|
||||
});
|
||||
|
||||
// Recommendations
|
||||
console.log('\n💡 RECOMMENDATIONS\n');
|
||||
console.log('• Playwright: Best for comprehensive testing with excellent debugging');
|
||||
console.log('• Puppeteer: Best for Chrome-specific features and CDP access');
|
||||
console.log('• Agent-Browser: Best for AI agents with semantic navigation needs');
|
||||
|
||||
console.log('\n' + '='.repeat(70) + '\n');
|
||||
}
|
||||
|
||||
// Export results for external analysis
|
||||
export { results as comparisonResults };
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Playwright Direct API Test Suite
|
||||
*
|
||||
* Tests Claudeman web UI using Playwright's native API directly.
|
||||
* This provides the most control and best debugging capabilities.
|
||||
*
|
||||
* Port: 3155 (see CLAUDE.md test port table)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { chromium, Browser, Page, BrowserContext } from 'playwright';
|
||||
import { WebServer } from '../src/web/server.js';
|
||||
|
||||
const TEST_PORT = 3155;
|
||||
const BASE_URL = `http://localhost:${TEST_PORT}`;
|
||||
|
||||
// Metrics collection
|
||||
interface TestMetrics {
|
||||
testName: string;
|
||||
duration: number;
|
||||
framework: 'playwright';
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const metrics: TestMetrics[] = [];
|
||||
let server: WebServer;
|
||||
let browser: Browser;
|
||||
let createdSessions: string[] = [];
|
||||
let setupComplete = false;
|
||||
|
||||
describe('Playwright Direct API Tests', () => {
|
||||
beforeAll(async () => {
|
||||
// Start web server
|
||||
server = new WebServer(TEST_PORT);
|
||||
await server.start();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Launch browser
|
||||
browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
|
||||
});
|
||||
setupComplete = true;
|
||||
}, 30000);
|
||||
|
||||
afterAll(async () => {
|
||||
// Cleanup sessions
|
||||
for (const sessionId of createdSessions) {
|
||||
try {
|
||||
await fetch(`${BASE_URL}/api/sessions/${sessionId}`, { method: 'DELETE' });
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Close browser and server
|
||||
if (browser) await browser.close();
|
||||
if (server) await server.stop();
|
||||
|
||||
// Output metrics
|
||||
console.log('\n📊 Playwright Test Metrics:');
|
||||
console.table(metrics.map(m => ({
|
||||
test: m.testName.substring(0, 40),
|
||||
duration: `${m.duration}ms`,
|
||||
status: m.success ? '✅' : '❌',
|
||||
})));
|
||||
}, 30000);
|
||||
|
||||
// Helper to run test with metrics
|
||||
async function withMetrics(name: string, fn: (page: Page) => Promise<void>) {
|
||||
if (!setupComplete) {
|
||||
console.log('Skipping - setup not complete');
|
||||
return;
|
||||
}
|
||||
const start = Date.now();
|
||||
let success = false;
|
||||
let error: string | undefined;
|
||||
const context = await browser.newContext({ viewport: { width: 1280, height: 720 } });
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
await fn(page);
|
||||
success = true;
|
||||
} catch (e: any) {
|
||||
error = e.message;
|
||||
throw e;
|
||||
} finally {
|
||||
await context.close();
|
||||
metrics.push({ testName: name, duration: Date.now() - start, framework: 'playwright', success, error });
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Core Page Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should load the main page with correct title', async () => {
|
||||
await withMetrics('page-load', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
await expect(page).toHaveTitle('Claudeman');
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it('should display the logo and branding', async () => {
|
||||
await withMetrics('logo-branding', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
const logo = page.locator('.header-brand .logo');
|
||||
await expect(logo).toBeVisible();
|
||||
await expect(logo).toHaveText('Claudeman');
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it('should show version in toolbar', async () => {
|
||||
await withMetrics('version-display', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
const version = page.locator('#versionDisplay');
|
||||
await expect(version).toBeVisible();
|
||||
await expect(version).toHaveText(/v\d+\.\d+/);
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// Modal Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should open and close help modal', async () => {
|
||||
await withMetrics('help-modal', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
await page.click('.help-btn');
|
||||
const modal = page.locator('#helpModal .modal-content');
|
||||
await expect(modal).toBeVisible();
|
||||
const heading = page.locator('#helpModal h3');
|
||||
await expect(heading).toContainText('Keyboard Shortcuts');
|
||||
await page.click('#helpModal .modal-close');
|
||||
await expect(modal).not.toBeVisible();
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it('should open settings modal with 4 tabs', async () => {
|
||||
await withMetrics('settings-modal', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
await page.click('.btn-settings');
|
||||
const modal = page.locator('#appSettingsModal .modal-content');
|
||||
await expect(modal).toBeVisible();
|
||||
const tabs = page.locator('#appSettingsModal .modal-tab-btn');
|
||||
await expect(tabs).toHaveCount(4);
|
||||
await page.click('#appSettingsModal .modal-close');
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// Session Management Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should create a new session via button click', async () => {
|
||||
await withMetrics('session-creation', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
await page.click('.btn-claude');
|
||||
const sessionTab = page.locator('.session-tab');
|
||||
await expect(sessionTab.first()).toBeVisible({ timeout: 15000 });
|
||||
await expect(page.locator('.session-tab.active')).toBeVisible();
|
||||
const terminal = page.locator('.xterm');
|
||||
await expect(terminal).toBeVisible({ timeout: 10000 });
|
||||
|
||||
// Track session for cleanup
|
||||
const response = await fetch(`${BASE_URL}/api/sessions`);
|
||||
const data = await response.json();
|
||||
if (data.sessions?.length > 0) {
|
||||
const lastSession = data.sessions[data.sessions.length - 1];
|
||||
if (lastSession && !createdSessions.includes(lastSession.id)) {
|
||||
createdSessions.push(lastSession.id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}, 60000);
|
||||
|
||||
// ============================================================================
|
||||
// Stress Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should handle rapid modal interactions', async () => {
|
||||
await withMetrics('rapid-interactions', async (page) => {
|
||||
await page.goto(BASE_URL);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'visible' });
|
||||
await page.click('#helpModal .modal-close');
|
||||
await page.waitForSelector('#helpModal .modal-content', { state: 'hidden' });
|
||||
}
|
||||
await expect(page.locator('.header')).toBeVisible();
|
||||
});
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
// Export metrics for comparison
|
||||
export { metrics as playwrightMetrics };
|
||||
@@ -0,0 +1,445 @@
|
||||
/**
|
||||
* Puppeteer/Chrome Test Suite
|
||||
*
|
||||
* Tests Claudeman web UI using Puppeteer for direct Chrome DevTools Protocol access.
|
||||
* Puppeteer provides lower-level Chrome control compared to Playwright.
|
||||
*
|
||||
* Port: 3156 (see CLAUDE.md test port table)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import puppeteer, { Browser, Page } from 'puppeteer';
|
||||
import { WebServer } from '../src/web/server.js';
|
||||
|
||||
const TEST_PORT = 3156;
|
||||
const BASE_URL = `http://localhost:${TEST_PORT}`;
|
||||
|
||||
// Metrics collection
|
||||
interface TestMetrics {
|
||||
testName: string;
|
||||
duration: number;
|
||||
framework: 'puppeteer';
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const metrics: TestMetrics[] = [];
|
||||
|
||||
describe('Puppeteer/Chrome Tests', () => {
|
||||
let server: WebServer;
|
||||
let browser: Browser;
|
||||
let page: Page;
|
||||
let createdSessions: string[] = [];
|
||||
let testStartTime: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Start web server
|
||||
server = new WebServer(TEST_PORT);
|
||||
await server.start();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Launch Chrome via Puppeteer
|
||||
browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
],
|
||||
});
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
// Cleanup sessions
|
||||
for (const sessionId of createdSessions) {
|
||||
try {
|
||||
await fetch(`${BASE_URL}/api/sessions/${sessionId}`, { method: 'DELETE' });
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Close browser and server
|
||||
if (browser) await browser.close();
|
||||
await server.stop();
|
||||
|
||||
// Output metrics
|
||||
console.log('\n📊 Puppeteer Test Metrics:');
|
||||
console.table(metrics.map(m => ({
|
||||
test: m.testName.substring(0, 40),
|
||||
duration: `${m.duration}ms`,
|
||||
status: m.success ? '✅' : '❌',
|
||||
})));
|
||||
}, 60000);
|
||||
|
||||
beforeEach(async () => {
|
||||
page = await browser.newPage();
|
||||
await page.setViewport({ width: 1280, height: 720 });
|
||||
testStartTime = Date.now();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
const testInfo = expect.getState();
|
||||
metrics.push({
|
||||
testName: testInfo.currentTestName || 'unknown',
|
||||
duration: Date.now() - testStartTime,
|
||||
framework: 'puppeteer',
|
||||
success: !testInfo.assertionCalls?.some((c: any) => !c.pass),
|
||||
});
|
||||
|
||||
if (page) await page.close();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Core Page Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should load the main page with correct title', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
const title = await page.title();
|
||||
expect(title).toBe('Claudeman');
|
||||
});
|
||||
|
||||
it('should display the logo and branding', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
await page.waitForSelector('.header-brand .logo');
|
||||
const logoText = await page.$eval('.header-brand .logo', el => el.textContent);
|
||||
expect(logoText).toBe('Claudeman');
|
||||
});
|
||||
|
||||
it('should show version in toolbar', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
await page.waitForSelector('#versionDisplay');
|
||||
const version = await page.$eval('#versionDisplay', el => el.textContent);
|
||||
expect(version).toMatch(/v\d+\.\d+/);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Button & Control Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should have all main control buttons visible', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
const btnClaude = await page.$('.btn-claude');
|
||||
const btnSettings = await page.$('.btn-settings');
|
||||
const helpBtn = await page.$('.help-btn');
|
||||
const quickStart = await page.$('#quickStartCase');
|
||||
|
||||
expect(btnClaude).not.toBeNull();
|
||||
expect(btnSettings).not.toBeNull();
|
||||
expect(helpBtn).not.toBeNull();
|
||||
expect(quickStart).not.toBeNull();
|
||||
});
|
||||
|
||||
it('should have font size controls', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
const fontControls = await page.$('.header-font-controls');
|
||||
expect(fontControls).not.toBeNull();
|
||||
|
||||
const fontBtns = await page.$$('.header-font-controls .font-btn');
|
||||
expect(fontBtns.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Modal Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should open and close help modal', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Open help modal
|
||||
await page.click('.help-btn');
|
||||
await page.waitForSelector('#helpModal .modal-content', { visible: true });
|
||||
|
||||
// Check content
|
||||
const heading = await page.$eval('#helpModal h3', el => el.textContent);
|
||||
expect(heading).toContain('Keyboard Shortcuts');
|
||||
|
||||
// Check modal is visible
|
||||
const isVisible = await page.$eval('#helpModal .modal-content', el => {
|
||||
const style = window.getComputedStyle(el);
|
||||
return style.display !== 'none' && style.visibility !== 'hidden';
|
||||
});
|
||||
expect(isVisible).toBe(true);
|
||||
|
||||
// Close modal
|
||||
await page.click('#helpModal .modal-close');
|
||||
await page.waitForFunction(() => {
|
||||
const modal = document.querySelector('#helpModal .modal-content');
|
||||
if (!modal) return true;
|
||||
const style = window.getComputedStyle(modal);
|
||||
return style.display === 'none' || style.visibility === 'hidden';
|
||||
}, { timeout: 5000 }).catch(() => {});
|
||||
});
|
||||
|
||||
it('should open settings modal with 4 tabs', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Open settings
|
||||
await page.click('.btn-settings');
|
||||
await page.waitForSelector('#appSettingsModal .modal-content', { visible: true });
|
||||
|
||||
// Check tabs
|
||||
const tabs = await page.$$('#appSettingsModal .modal-tab-btn');
|
||||
expect(tabs.length).toBe(4);
|
||||
|
||||
// Verify tab names
|
||||
const tabTexts = await page.$$eval('#appSettingsModal .modal-tab-btn', els =>
|
||||
els.map(el => el.textContent?.trim())
|
||||
);
|
||||
expect(tabTexts).toContain('Display');
|
||||
expect(tabTexts).toContain('Claude CLI');
|
||||
|
||||
// Close
|
||||
await page.click('#appSettingsModal .modal-close');
|
||||
});
|
||||
|
||||
it('should switch between settings tabs', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
await page.click('.btn-settings');
|
||||
await page.waitForSelector('#appSettingsModal .modal-content', { visible: true });
|
||||
|
||||
// Click each tab
|
||||
const tabs = await page.$$('#appSettingsModal .modal-tab-btn');
|
||||
|
||||
for (let i = 0; i < tabs.length; i++) {
|
||||
await tabs[i].click();
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
|
||||
// Check tab is active
|
||||
const isActive = await page.$eval(
|
||||
`#appSettingsModal .modal-tab-btn:nth-child(${i + 1})`,
|
||||
el => el.classList.contains('active')
|
||||
);
|
||||
expect(isActive).toBe(true);
|
||||
}
|
||||
|
||||
await page.click('#appSettingsModal .modal-close');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Session Management Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should create a new session via button click', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Click create session button
|
||||
await page.click('.btn-claude');
|
||||
|
||||
// Wait for session tab to appear
|
||||
await page.waitForSelector('.session-tab', { timeout: 15000 });
|
||||
|
||||
// Tab should be active
|
||||
const hasActive = await page.$('.session-tab.active');
|
||||
expect(hasActive).not.toBeNull();
|
||||
|
||||
// Terminal should be visible
|
||||
await page.waitForSelector('.xterm', { timeout: 10000 });
|
||||
const terminal = await page.$('.xterm');
|
||||
expect(terminal).not.toBeNull();
|
||||
|
||||
// Track session for cleanup
|
||||
const response = await fetch(`${BASE_URL}/api/sessions`);
|
||||
const data = await response.json();
|
||||
if (data.sessions?.length > 0) {
|
||||
const lastSession = data.sessions[data.sessions.length - 1];
|
||||
if (lastSession && !createdSessions.includes(lastSession.id)) {
|
||||
createdSessions.push(lastSession.id);
|
||||
}
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
it('should show tab count correctly', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Get initial count
|
||||
const initialText = await page.$eval('#tabCount', el => el.textContent);
|
||||
const initialCount = parseInt(initialText?.match(/\d+/)?.[0] || '0');
|
||||
|
||||
// Create a session
|
||||
await page.click('.btn-claude');
|
||||
await page.waitForSelector('.session-tab', { timeout: 15000 });
|
||||
|
||||
// Tab count should increase
|
||||
await page.waitForFunction(
|
||||
(expected: number) => {
|
||||
const el = document.querySelector('#tabCount');
|
||||
const text = el?.textContent || '';
|
||||
const match = text.match(/\d+/);
|
||||
return match && parseInt(match[0]) >= expected;
|
||||
},
|
||||
{ timeout: 5000 },
|
||||
initialCount + 1
|
||||
);
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`${BASE_URL}/api/sessions`);
|
||||
const data = await response.json();
|
||||
data.sessions?.forEach((s: any) => {
|
||||
if (!createdSessions.includes(s.id)) createdSessions.push(s.id);
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// Terminal Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should initialize xterm terminal with correct dimensions', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Create session
|
||||
await page.click('.btn-claude');
|
||||
await page.waitForSelector('.xterm', { timeout: 15000 });
|
||||
|
||||
// Check terminal container exists
|
||||
const terminal = await page.$('.xterm');
|
||||
expect(terminal).not.toBeNull();
|
||||
|
||||
// Check xterm canvas exists (indicates proper initialization)
|
||||
const canvas = await page.$('.xterm-screen');
|
||||
expect(canvas).not.toBeNull();
|
||||
|
||||
// Track for cleanup
|
||||
const response = await fetch(`${BASE_URL}/api/sessions`);
|
||||
const data = await response.json();
|
||||
data.sessions?.forEach((s: any) => {
|
||||
if (!createdSessions.includes(s.id)) createdSessions.push(s.id);
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// Keyboard Shortcut Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should respond to Ctrl+Enter for quick start', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
const initialSessions = await fetch(`${BASE_URL}/api/sessions`).then(r => r.json());
|
||||
const initialCount = initialSessions.sessions?.length || 0;
|
||||
|
||||
// Press Ctrl+Enter
|
||||
await page.keyboard.down('Control');
|
||||
await page.keyboard.press('Enter');
|
||||
await page.keyboard.up('Control');
|
||||
|
||||
// Wait for session to be created
|
||||
await page.waitForSelector('.session-tab', { timeout: 15000 });
|
||||
|
||||
// Verify session was created
|
||||
const afterSessions = await fetch(`${BASE_URL}/api/sessions`).then(r => r.json());
|
||||
expect(afterSessions.sessions?.length).toBeGreaterThan(initialCount);
|
||||
|
||||
// Track for cleanup
|
||||
afterSessions.sessions?.forEach((s: any) => {
|
||||
if (!createdSessions.includes(s.id)) createdSessions.push(s.id);
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
// ============================================================================
|
||||
// Chrome DevTools Protocol Direct Access
|
||||
// ============================================================================
|
||||
|
||||
it('should access CDP for performance metrics', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Get performance metrics via CDP
|
||||
const client = await page.target().createCDPSession();
|
||||
await client.send('Performance.enable');
|
||||
|
||||
const { metrics: perfMetrics } = await client.send('Performance.getMetrics');
|
||||
|
||||
// Should have some metrics
|
||||
expect(perfMetrics.length).toBeGreaterThan(0);
|
||||
|
||||
// Find specific metrics
|
||||
const jsHeapSize = perfMetrics.find(m => m.name === 'JSHeapUsedSize');
|
||||
expect(jsHeapSize).toBeDefined();
|
||||
});
|
||||
|
||||
it('should capture network activity via CDP', async () => {
|
||||
const networkRequests: string[] = [];
|
||||
|
||||
const client = await page.target().createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
|
||||
client.on('Network.requestWillBeSent', (params) => {
|
||||
networkRequests.push(params.request.url);
|
||||
});
|
||||
|
||||
await page.goto(BASE_URL);
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
|
||||
// Should have captured some requests
|
||||
expect(networkRequests.length).toBeGreaterThan(0);
|
||||
expect(networkRequests.some(url => url.includes('/api/events'))).toBe(true);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Stress Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should handle rapid button clicks without errors', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Rapidly click help button multiple times
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await page.click('.help-btn');
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
const closeBtn = await page.$('#helpModal .modal-close');
|
||||
if (closeBtn) await closeBtn.click().catch(() => {});
|
||||
await new Promise(r => setTimeout(r, 100));
|
||||
}
|
||||
|
||||
// Page should still be functional
|
||||
const header = await page.$('.header');
|
||||
expect(header).not.toBeNull();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Screenshot & Visual Tests
|
||||
// ============================================================================
|
||||
|
||||
it('should capture screenshot for visual inspection', async () => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Take a screenshot (could be used for visual regression)
|
||||
const screenshot = await page.screenshot({ encoding: 'base64' });
|
||||
expect(screenshot.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Console & Error Monitoring
|
||||
// ============================================================================
|
||||
|
||||
it('should not have console errors on page load', async () => {
|
||||
const consoleErrors: string[] = [];
|
||||
|
||||
page.on('console', msg => {
|
||||
if (msg.type() === 'error') {
|
||||
consoleErrors.push(msg.text());
|
||||
}
|
||||
});
|
||||
|
||||
page.on('pageerror', err => {
|
||||
consoleErrors.push(err.message);
|
||||
});
|
||||
|
||||
await page.goto(BASE_URL);
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
|
||||
// Filter out expected errors (like SSE reconnection)
|
||||
const unexpectedErrors = consoleErrors.filter(e =>
|
||||
!e.includes('net::ERR') && !e.includes('SSE')
|
||||
);
|
||||
|
||||
expect(unexpectedErrors.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// Export metrics for comparison
|
||||
export { metrics as puppeteerMetrics };
|
||||
Reference in New Issue
Block a user