mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
fix(mobile): route terminal content taps to the CLI
This commit is contained in:
+131
-52
@@ -650,7 +650,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
|
||||
let didScroll = false; // track whether touchmove fired (tap vs scroll)
|
||||
let touchStartY = 0;
|
||||
let tapCanActivateTerminal = false;
|
||||
let tapStartedWithTerminalFocus = false;
|
||||
const TAP_THRESHOLD = 8; // px — ignore micro-drift to distinguish tap from scroll
|
||||
container.addEventListener(
|
||||
'touchstart',
|
||||
@@ -663,7 +663,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
pixelAccum = 0;
|
||||
isTouching = true;
|
||||
didScroll = false;
|
||||
tapCanActivateTerminal = this._shouldForwardMobileTapToApp();
|
||||
tapStartedWithTerminalFocus = this._isMobileTerminalInputFocused();
|
||||
lastTime = 0;
|
||||
if (scrollFrame) {
|
||||
cancelAnimationFrame(scrollFrame);
|
||||
@@ -723,49 +723,13 @@ Object.assign(CodemanApp.prototype, {
|
||||
scrollFrame = requestAnimationFrame(scrollLoop);
|
||||
}
|
||||
if (!didScroll && this.terminal) {
|
||||
// ── Tap-to-position cursor ──────────────────────────────────
|
||||
// Synthesize a click from the real touch point so the foreground app
|
||||
// moves its cursor to the tapped cell (iOS doesn't reliably do this
|
||||
// itself under touch-action:none). CRITICAL: only when mouse tracking
|
||||
// is ON. xterm disables its local SelectionService while mouse events
|
||||
// are active, so the synthetic click is forwarded to the PTY as an SGR
|
||||
// report (cursor moves). But when tracking is OFF, that same click
|
||||
// drives xterm's LOCAL selection (detail 1/2/3 → char/word/line) — a
|
||||
// tap on CJK text would select & copy it instead of positioning. So
|
||||
// gate strictly on the live mouse-tracking mode.
|
||||
const touch = ev.changedTouches && ev.changedTouches[0];
|
||||
const mouseMode = this.terminal.modes?.mouseTrackingMode;
|
||||
const mouseTrackingOn = !!mouseMode && mouseMode !== 'none';
|
||||
if (touch) {
|
||||
this._suppressTrustedTapMouseEvents();
|
||||
}
|
||||
if (touch && tapCanActivateTerminal && mouseTrackingOn) {
|
||||
this._dispatchSyntheticTerminalClick(touch.clientX, touch.clientY);
|
||||
} else if (
|
||||
touch &&
|
||||
tapCanActivateTerminal &&
|
||||
this._sessionUsesServerMouseStrip()
|
||||
) {
|
||||
// The server strips mouse-tracking DECSETs from claude/codex/gemini
|
||||
// output (isAltScreenStripMode, session.ts) so the wheel keeps
|
||||
// scrolling scrollback — which leaves THIS xterm permanently at
|
||||
// mouseTrackingMode 'none' even though the TUI on the PTY side has
|
||||
// tracking ON and still understands SGR reports. Encode the report
|
||||
// ourselves and send it straight to the PTY: no DOM click is
|
||||
// dispatched, so xterm's local selection can't trigger either.
|
||||
this._sendSyntheticSgrTap(touch.clientX, touch.clientY);
|
||||
}
|
||||
this._syncMobileHelperTextareaToCursor();
|
||||
// Route subsequent typing to the right place: keep the CJK input
|
||||
// field focused when Chinese input is on, otherwise the terminal.
|
||||
const cjkInput = document.getElementById('cjkInput');
|
||||
if (cjkInput?.classList.contains('cjk-input-visible')) {
|
||||
cjkInput.focus();
|
||||
} else {
|
||||
this.terminal.focus();
|
||||
this._handleMobileTerminalTap(touch, tapStartedWithTerminalFocus);
|
||||
}
|
||||
}
|
||||
tapCanActivateTerminal = false;
|
||||
tapStartedWithTerminalFocus = false;
|
||||
},
|
||||
{ passive: true }
|
||||
);
|
||||
@@ -776,6 +740,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
isTouching = false;
|
||||
velocity = 0;
|
||||
pixelAccum = 0;
|
||||
tapStartedWithTerminalFocus = false;
|
||||
},
|
||||
{ passive: true }
|
||||
);
|
||||
@@ -3370,18 +3335,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
} catch {}
|
||||
},
|
||||
|
||||
/**
|
||||
* A tap that opens the phone keyboard is focus-only. Forwarding that same
|
||||
* gesture as a mouse click would activate the highlighted CLI menu option.
|
||||
* Once the keyboard and terminal input were already active at touchstart,
|
||||
* later taps may intentionally position the cursor or select a TUI row.
|
||||
*/
|
||||
_shouldForwardMobileTapToApp() {
|
||||
const keyboardVisible =
|
||||
(typeof KeyboardHandler !== 'undefined' && KeyboardHandler.keyboardVisible) ||
|
||||
document.body?.classList?.contains('keyboard-visible');
|
||||
if (!keyboardVisible) return false;
|
||||
|
||||
_isMobileTerminalInputFocused() {
|
||||
const active = document.activeElement;
|
||||
return (
|
||||
active === this.terminal?.textarea ||
|
||||
@@ -3390,6 +3344,131 @@ Object.assign(CodemanApp.prototype, {
|
||||
);
|
||||
},
|
||||
|
||||
/**
|
||||
* Separate terminal input from TUI-owned content on touch devices. A hidden
|
||||
* keyboard must not consume taps on expandable readbacks, tool results, or
|
||||
* decision rows; those taps belong to the foreground CLI. The visible prompt
|
||||
* row remains the deliberate keyboard target.
|
||||
*/
|
||||
_classifyMobileTerminalTap(clientX, clientY) {
|
||||
if (!this._terminalViewportAtBottom()) return 'history';
|
||||
|
||||
const pos = this._clientPointToCell(clientX, clientY);
|
||||
if (!pos || !this.terminal) return 'input';
|
||||
|
||||
const mouseMode = this.terminal.modes?.mouseTrackingMode;
|
||||
const mouseTrackingOn = !!mouseMode && mouseMode !== 'none';
|
||||
if (!mouseTrackingOn && !this._sessionUsesServerMouseStrip()) return 'input';
|
||||
|
||||
// Permission/elicitation prompts own the full live terminal until answered.
|
||||
if (document.body?.classList?.contains('terminal-action-pending')) return 'content';
|
||||
|
||||
const buffer = this.terminal.buffer?.active;
|
||||
if (!buffer?.getLine) return 'input';
|
||||
|
||||
const rows = Math.max(1, this.terminal.rows || 1);
|
||||
const lines = [];
|
||||
let hasVisibleContent = false;
|
||||
for (let row = 0; row < rows; row++) {
|
||||
const line = buffer.getLine(buffer.viewportY + row);
|
||||
const text = line?.translateToString?.(true) || '';
|
||||
lines.push(text);
|
||||
if (text.trim()) hasVisibleContent = true;
|
||||
}
|
||||
if (!hasVisibleContent) return 'input';
|
||||
|
||||
const cursorRow = Math.max(0, Math.min(rows - 1, buffer.cursorY || 0));
|
||||
const mode = this.sessions?.get(this.activeSessionId)?.mode || 'claude';
|
||||
let promptRow = -1;
|
||||
let menuSelectionVisible = false;
|
||||
|
||||
if (mode === 'opencode') {
|
||||
if (lines[cursorRow]?.includes('\u2503')) promptRow = cursorRow;
|
||||
} else {
|
||||
for (let row = rows - 1; row >= 0; row--) {
|
||||
const promptMatch = lines[row].match(/^\s*[❯›]/);
|
||||
if (!promptMatch) continue;
|
||||
const tail = lines[row].slice(promptMatch[0].length).trim();
|
||||
// A highlighted numbered choice is a menu row, not an editable prompt.
|
||||
const hasSiblingChoice = lines.some(
|
||||
(line, choiceRow) => choiceRow !== row && /^\s+\d+[.)]\s/.test(line)
|
||||
);
|
||||
if (/^\d+[.)]\s/.test(tail) && hasSiblingChoice) {
|
||||
menuSelectionVisible = true;
|
||||
break;
|
||||
}
|
||||
promptRow = row;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const tappedRow = pos.row - 1;
|
||||
if (menuSelectionVisible) return 'content';
|
||||
if (promptRow >= 0) {
|
||||
const inputEnd = cursorRow >= promptRow ? cursorRow : promptRow;
|
||||
if (tappedRow >= promptRow && tappedRow <= inputEnd) return 'input';
|
||||
} else if (tappedRow === cursorRow && cursorRow >= rows - 3) {
|
||||
// During redraws a CLI can temporarily omit its prompt marker. Keep the
|
||||
// live cursor row usable without turning arbitrary transcript rows into
|
||||
// keyboard targets.
|
||||
return 'input';
|
||||
}
|
||||
|
||||
return 'content';
|
||||
},
|
||||
|
||||
_blurMobileTerminalInput() {
|
||||
const active = document.activeElement;
|
||||
if (
|
||||
active === this.terminal?.textarea ||
|
||||
active?.classList?.contains('xterm-helper-textarea') ||
|
||||
active?.id === 'cjkInput'
|
||||
) {
|
||||
active.blur?.();
|
||||
}
|
||||
},
|
||||
|
||||
_focusMobileTerminalInput() {
|
||||
this._syncMobileHelperTextareaToCursor();
|
||||
const cjkInput = document.getElementById('cjkInput');
|
||||
if (cjkInput?.classList.contains('cjk-input-visible')) {
|
||||
cjkInput.focus();
|
||||
} else {
|
||||
this.terminal?.focus();
|
||||
}
|
||||
},
|
||||
|
||||
_handleMobileTerminalTap(touch, startedWithTerminalFocus) {
|
||||
if (!touch || !this.terminal) return 'history';
|
||||
const intent = this._classifyMobileTerminalTap(touch.clientX, touch.clientY);
|
||||
if (intent === 'history') {
|
||||
this._blurMobileTerminalInput();
|
||||
return intent;
|
||||
}
|
||||
|
||||
const mouseMode = this.terminal.modes?.mouseTrackingMode;
|
||||
const mouseTrackingOn = !!mouseMode && mouseMode !== 'none';
|
||||
const shouldActivate = intent === 'content' || startedWithTerminalFocus;
|
||||
if (shouldActivate && mouseTrackingOn) {
|
||||
// xterm's mouse encoder owns live DECSET modes. The synthetic DOM click
|
||||
// follows the same path as a desktop click.
|
||||
this._dispatchSyntheticTerminalClick(touch.clientX, touch.clientY);
|
||||
} else if (shouldActivate && this._sessionUsesServerMouseStrip()) {
|
||||
// Claude/Codex/Gemini DECSETs are stripped from the browser stream, so
|
||||
// report directly to the PTY while retaining local touch scrollback.
|
||||
this._sendSyntheticSgrTap(touch.clientX, touch.clientY);
|
||||
}
|
||||
|
||||
if (intent === 'content') {
|
||||
// A synthetic xterm click can focus its helper textarea. Blur after the
|
||||
// report so collapsing a readback never opens or retains the keyboard.
|
||||
this._blurMobileTerminalInput();
|
||||
} else {
|
||||
this._focusMobileTerminalInput();
|
||||
}
|
||||
return intent;
|
||||
},
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Synthetic tap → mouse report
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
@@ -754,8 +754,53 @@ describe('Virtual Keyboard', () => {
|
||||
expect(state?.bodyClass).toBe(false);
|
||||
});
|
||||
|
||||
it('focuses the terminal helper textarea when the terminal is tapped', async () => {
|
||||
await page.evaluate(() => {
|
||||
it('collapses a terminal readback without focusing the hidden textarea', async () => {
|
||||
const point = await page.evaluate(async () => {
|
||||
window.__sentInputs = [];
|
||||
app.activeSessionId = 'mobile-readback-tap-test';
|
||||
app.sessions.set('mobile-readback-tap-test', {
|
||||
id: 'mobile-readback-tap-test',
|
||||
mode: 'codex',
|
||||
status: 'running',
|
||||
});
|
||||
app._sendInputAsync = (_sessionId: string, input: string) => {
|
||||
window.__sentInputs.push(input);
|
||||
};
|
||||
app.hideWelcome();
|
||||
const settings = app.loadAppSettingsFromStorage();
|
||||
settings.cjkInputEnabled = false;
|
||||
app.saveAppSettingsToStorage(settings);
|
||||
app._updateCjkInputState();
|
||||
app.terminal.reset();
|
||||
await new Promise<void>((resolve) =>
|
||||
app.terminal.write('Agent readback\r\n tap to collapse\r\n\r\n› ask', resolve)
|
||||
);
|
||||
app.terminal.focus();
|
||||
|
||||
const screen = app.terminal.element?.querySelector('.xterm-screen');
|
||||
const cell = app.terminal._core?._renderService?.dimensions?.css?.cell;
|
||||
const rect = screen?.getBoundingClientRect();
|
||||
if (!rect || !cell?.width || !cell?.height) return null;
|
||||
return {
|
||||
x: rect.left + cell.width * 2,
|
||||
y: rect.top + cell.height / 2,
|
||||
};
|
||||
});
|
||||
expect(point).not.toBeNull();
|
||||
|
||||
await page.touchscreen.tap(point!.x, point!.y);
|
||||
|
||||
const state = await page.evaluate(() => ({
|
||||
activeClass: document.activeElement?.className,
|
||||
sentInputs: window.__sentInputs,
|
||||
}));
|
||||
expect(state.activeClass).not.toContain('xterm-helper-textarea');
|
||||
expect(state.sentInputs).toHaveLength(1);
|
||||
expect(state.sentInputs[0]).toMatch(/^\x1b\[<0;\d+;1M\x1b\[<0;\d+;1m$/);
|
||||
});
|
||||
|
||||
it('focuses the terminal helper textarea when the visible prompt is tapped', async () => {
|
||||
const point = await page.evaluate(async () => {
|
||||
window.__sentInputs = [];
|
||||
app.activeSessionId = 'mobile-focus-visible-input-test';
|
||||
app.sessions.set('mobile-focus-visible-input-test', {
|
||||
@@ -771,9 +816,24 @@ describe('Virtual Keyboard', () => {
|
||||
settings.cjkInputEnabled = false;
|
||||
app.saveAppSettingsToStorage(settings);
|
||||
app._updateCjkInputState();
|
||||
});
|
||||
app.terminal.reset();
|
||||
await new Promise<void>((resolve) =>
|
||||
app.terminal.write('Agent readback\r\n tap to collapse\r\n\r\n› ask', resolve)
|
||||
);
|
||||
(document.activeElement as HTMLElement | null)?.blur?.();
|
||||
|
||||
await page.locator('#terminalContainer').tap({ position: { x: 40, y: 40 } });
|
||||
const screen = app.terminal.element?.querySelector('.xterm-screen');
|
||||
const cell = app.terminal._core?._renderService?.dimensions?.css?.cell;
|
||||
const rect = screen?.getBoundingClientRect();
|
||||
if (!rect || !cell?.width || !cell?.height) return null;
|
||||
return {
|
||||
x: rect.left + cell.width * 2,
|
||||
y: rect.top + cell.height * (app.terminal.buffer.active.cursorY + 0.5),
|
||||
};
|
||||
});
|
||||
expect(point).not.toBeNull();
|
||||
|
||||
await page.touchscreen.tap(point!.x, point!.y);
|
||||
|
||||
const state = await page.evaluate(() => ({
|
||||
activeClass: document.activeElement?.className,
|
||||
|
||||
@@ -15,6 +15,7 @@ function loadTerminalUiHarness() {
|
||||
get activeElement() {
|
||||
return activeElement;
|
||||
},
|
||||
getElementById: () => null,
|
||||
},
|
||||
CodemanApp,
|
||||
console: { warn: vi.fn(), log: vi.fn() },
|
||||
@@ -74,18 +75,103 @@ function createElementHarness() {
|
||||
};
|
||||
}
|
||||
|
||||
function createTerminalGrid(lines: string[], cursorY: number) {
|
||||
const textarea = {
|
||||
classList: { contains: (name: string) => name === 'xterm-helper-textarea' },
|
||||
blur: vi.fn(),
|
||||
};
|
||||
return {
|
||||
cols: 80,
|
||||
rows: lines.length,
|
||||
modes: { mouseTrackingMode: 'none' },
|
||||
buffer: {
|
||||
active: {
|
||||
viewportY: 0,
|
||||
baseY: 0,
|
||||
cursorY,
|
||||
getLine: (row: number) =>
|
||||
row >= 0 && row < lines.length ? { translateToString: () => lines[row] } : undefined,
|
||||
},
|
||||
},
|
||||
element: {
|
||||
querySelector: (selector: string) =>
|
||||
selector === '.xterm-screen' ? { getBoundingClientRect: () => ({ left: 0, top: 0 }) } : null,
|
||||
},
|
||||
_core: { _renderService: { dimensions: { css: { cell: { width: 8, height: 16 } } } } },
|
||||
textarea,
|
||||
focus: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
describe('terminal touch tap mouse guard', () => {
|
||||
it('keeps the keyboard-opening tap focus-only', () => {
|
||||
const { app, setActiveElement, setKeyboardVisible } = loadTerminalUiHarness();
|
||||
it('recognizes focus only when a terminal input owns the active element', () => {
|
||||
const { app, setActiveElement } = loadTerminalUiHarness();
|
||||
const textarea = { classList: { contains: () => true } };
|
||||
app.terminal = { textarea };
|
||||
|
||||
setActiveElement(null);
|
||||
expect(app._isMobileTerminalInputFocused()).toBe(false);
|
||||
|
||||
setActiveElement(textarea);
|
||||
expect(app._isMobileTerminalInputFocused()).toBe(true);
|
||||
});
|
||||
|
||||
setKeyboardVisible(false);
|
||||
expect(app._shouldForwardMobileTapToApp()).toBe(false);
|
||||
it('routes a readback row to the TUI while keeping the prompt row as keyboard input', () => {
|
||||
const { app } = loadTerminalUiHarness();
|
||||
app.activeSessionId = 'sess-1';
|
||||
app.sessions = new Map([['sess-1', { mode: 'codex' }]]);
|
||||
app.terminal = createTerminalGrid(
|
||||
['Agent readback mentions › inline', ' tap to collapse', '', '', '› ask', 'gpt-5 · Context 80% left'],
|
||||
4
|
||||
);
|
||||
|
||||
setKeyboardVisible(true);
|
||||
expect(app._shouldForwardMobileTapToApp()).toBe(true);
|
||||
expect(app._classifyMobileTerminalTap(9, 1)).toBe('content'); // inline marker is not a prompt
|
||||
expect(app._classifyMobileTerminalTap(9, 17)).toBe('content'); // row 2: readback
|
||||
expect(app._classifyMobileTerminalTap(9, 65)).toBe('input'); // row 5: prompt
|
||||
expect(app._classifyMobileTerminalTap(9, 81)).toBe('content'); // row 6: status
|
||||
});
|
||||
|
||||
it('treats a highlighted numbered choice as TUI content, not an input prompt', () => {
|
||||
const { app } = loadTerminalUiHarness();
|
||||
app.activeSessionId = 'sess-1';
|
||||
app.sessions = new Map([['sess-1', { mode: 'claude' }]]);
|
||||
app.terminal = createTerminalGrid(['Would you like to proceed?', '', '❯ 1. Yes', ' 2. No', '', ''], 2);
|
||||
|
||||
expect(app._classifyMobileTerminalTap(9, 33)).toBe('content');
|
||||
expect(app._classifyMobileTerminalTap(9, 49)).toBe('content');
|
||||
});
|
||||
|
||||
it('collapses TUI readback content without opening or retaining the keyboard', () => {
|
||||
const { app, setActiveElement } = loadTerminalUiHarness();
|
||||
app.activeSessionId = 'sess-1';
|
||||
app.sessions = new Map([['sess-1', { mode: 'codex' }]]);
|
||||
app.terminal = createTerminalGrid(
|
||||
['Agent readback', ' tap to collapse', '', '', '› ask', 'gpt-5 · Context 80% left'],
|
||||
4
|
||||
);
|
||||
app._sendInputAsync = vi.fn();
|
||||
setActiveElement(app.terminal.textarea);
|
||||
|
||||
expect(app._handleMobileTerminalTap({ clientX: 9, clientY: 17 }, true)).toBe('content');
|
||||
expect(app._sendInputAsync).toHaveBeenCalledWith('sess-1', '\x1b[<0;2;2M\x1b[<0;2;2m');
|
||||
expect(app.terminal.textarea.blur).toHaveBeenCalledOnce();
|
||||
expect(app.terminal.focus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps the first prompt tap focus-only so it cannot activate a CLI row', () => {
|
||||
const { app, setActiveElement } = loadTerminalUiHarness();
|
||||
app.activeSessionId = 'sess-1';
|
||||
app.sessions = new Map([['sess-1', { mode: 'codex' }]]);
|
||||
app.terminal = createTerminalGrid(
|
||||
['Agent readback', ' tap to collapse', '', '', '› ask', 'gpt-5 · Context 80% left'],
|
||||
4
|
||||
);
|
||||
app._sendInputAsync = vi.fn();
|
||||
setActiveElement(null);
|
||||
|
||||
expect(app._handleMobileTerminalTap({ clientX: 9, clientY: 65 }, false)).toBe('input');
|
||||
expect(app._sendInputAsync).not.toHaveBeenCalled();
|
||||
expect(app.terminal.focus).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('suppresses browser trusted compatibility mouse events during the tap window', () => {
|
||||
|
||||
Reference in New Issue
Block a user