Merge pull request #499 from aakhter/pr/mobile-ime-preview

feat(terminal): preview IME composition text on iOS Safari

# Conflicts:
#	CLAUDE.md
This commit is contained in:
Codeman maintainer
2026-10-04 23:26:30 +02:00
15 changed files with 2565 additions and 14 deletions
+2
View File
@@ -3838,6 +3838,8 @@
<script defer src="notification-manager.js"></script>
<script defer src="keyboard-accessory.js"></script>
<script defer src="input-cjk.js"></script>
<!-- Shows iOS Safari IME composition text inside the terminal. Must precede terminal-ui.js. -->
<script defer src="mobile-ime-preview.js"></script>
<!-- Forwards committed input events that xterm drops on Android/GBoard soft keyboards. Must precede terminal-ui.js. -->
<script defer src="terminal-keycode229-recovery.js"></script>
<!-- Hardened markdown HTML sanitizer (wires DOMPurify). Must precede app.js. -->
+297
View File
@@ -0,0 +1,297 @@
/**
* @fileoverview In-terminal preview of IME composition text on iOS Safari.
*
* On iOS WebKit touch devices the text an IME is composing (Japanese, Chinese,
* Korean, dictation) is not visible inside the terminal until it commits, so
* the user types blind. The controller listens to the helper textarea's
* composition events and asks the caller to render the latest composition
* (`phase: 'provisional'`), coalesced to one render per animation frame and
* capped at 2048 characters. When xterm emits the committed text through
* onData, the caller hands it to `consumeTerminalData()`, which switches the
* preview to `phase: 'committed'` until something else shows the text: the
* local echo overlay or a prediction (`completeCommit`), authoritative
* terminal output (`noteAuthoritativeOutput`), or a 2 s fallback timer. The
* same 2 s bound applies while waiting for a commit that never reaches onData
* (the user deleted the whole composition), so a later unrelated chunk is never
* mistaken for it.
*
* Keydown ordering: xterm registers its textarea keydown listener in the
* capture phase inside terminal.open() and finalizes the composition there
* (CompositionHelper.keydown), emitting the commit through onData
* synchronously. The controller therefore observes keydown in the capture
* phase on an ANCESTOR (`keydownTarget`, the terminal element), which runs
* before any listener on the textarea itself, and finalizes on exactly the
* keys xterm does.
*
* VISUAL ONLY: the controller never sends, consumes or reorders input bytes,
* and every callback is wrapped so a failing render cannot block the wire.
* `isIosWebKitTouch()` gates creation; other platforms keep xterm's own
* composition view untouched.
*
* @dependency none (standalone IIFE; consumed by terminal-ui.js)
* @loadorder 5.52 (before app.js/terminal-ui.js, which create the controller)
*/
(function (global) {
'use strict';
const COMMITTED_VISUAL_TTL = 2000;
const PREVIEW_CAP = 2048;
// keyCodes on which xterm 6's CompositionHelper.keydown keeps composing
// (CapsLock, the IME "composition character", Shift/Ctrl/Alt). Any other
// keydown during a composition finalizes it.
const KEEP_COMPOSING_KEYCODES = new Set([20, 229, 16, 17, 18]);
const CONTROL_OR_LINE_BREAK = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/;
function isIosWebKitTouch(nav = navigator) {
const userAgent = String(nav && nav.userAgent ? nav.userAgent : '');
const platform = String(nav && nav.platform ? nav.platform : '');
const touchPoints = Number(nav && nav.maxTouchPoints ? nav.maxTouchPoints : 0);
const iosDevice = /iPhone|iPad|iPod/.test(userAgent);
const desktopIpad = platform === 'MacIntel' && touchPoints > 1;
return touchPoints > 0 && /AppleWebKit/.test(userAgent) && (iosDevice || desktopIpad);
}
function create(options) {
const textarea = options.textarea;
// Must be the textarea or an ancestor of it, so its capture listener runs
// before xterm's capture listener on the textarea.
const keydownTarget = options.keydownTarget || textarea;
const render = typeof options.render === 'function' ? options.render : function () {};
const clear = typeof options.clear === 'function' ? options.clear : function () {};
const onCommit = typeof options.onCommit === 'function' ? options.onCommit : function () {};
const scheduleFrame = options.scheduleFrame || global.requestAnimationFrame.bind(global);
const cancelFrame = options.cancelFrame || global.cancelAnimationFrame.bind(global);
const setTimer = options.setTimer || global.setTimeout.bind(global);
const clearTimer = options.clearTimer || global.clearTimeout.bind(global);
let generation = 0;
let composing = false;
let awaitingCommit = false;
let committed = false;
let latestValue = '';
let renderPhase = null;
let frameToken = null;
let timerToken = null;
let finalizedByKeydown = false;
let destroyed = false;
let invokingClear = false;
function safely(callback, ...args) {
try {
return callback(...args);
} catch (_error) {
return undefined;
}
}
function cancelScheduledFrame() {
const token = frameToken;
frameToken = null;
if (token && token.id !== undefined) safely(cancelFrame, token.id);
}
function cancelCommittedTimer() {
const token = timerToken;
timerToken = null;
if (token && token.id !== undefined) safely(clearTimer, token.id);
}
function clearVisual() {
if (invokingClear) return;
invokingClear = true;
safely(clear);
invokingClear = false;
}
function cleanup() {
generation += 1;
cancelScheduledFrame();
cancelCommittedTimer();
composing = false;
awaitingCommit = false;
committed = false;
latestValue = '';
renderPhase = null;
finalizedByKeydown = false;
clearVisual();
}
function scheduleLatestPreview(phase) {
if (destroyed) return;
renderPhase = phase;
if (frameToken) return;
const token = { generation, id: undefined };
frameToken = token;
const callback = function () {
if (destroyed || frameToken !== token || token.generation !== generation || renderPhase === null) return;
frameToken = null;
const value = latestValue.slice(0, PREVIEW_CAP);
const phaseToRender = renderPhase;
safely(render, { text: value, phase: phaseToRender });
};
const id = safely(scheduleFrame, callback);
if (frameToken === token) {
if (id === undefined) frameToken = null;
else token.id = id;
}
}
function beginComposition() {
cleanup();
if (destroyed) return;
composing = true;
}
function updateComposition(event) {
if (!composing) return;
latestValue = event.data == null ? '' : String(event.data);
scheduleLatestPreview('provisional');
}
function onComposingInput(event) {
if (!event.isComposing) return;
updateComposition({ data: event.data == null ? textarea.value : event.data });
}
function armFallbackTimer(isCurrent) {
const token = { generation, id: undefined };
timerToken = token;
const callback = function () {
if (destroyed || timerToken !== token || token.generation !== generation || !isCurrent()) return;
timerToken = null;
cleanup();
};
const id = safely(setTimer, callback, COMMITTED_VISUAL_TTL);
if (timerToken === token) {
if (id === undefined) {
timerToken = null;
return false;
}
token.id = id;
}
return true;
}
function finalizeComposition(value, fromKeydown) {
if (!composing) return;
composing = false;
awaitingCommit = true;
committed = false;
finalizedByKeydown = fromKeydown;
latestValue = value == null ? latestValue : String(value);
scheduleLatestPreview('provisional');
// A commit that never reaches onData (the composition was deleted, so
// xterm emits nothing) must not leave the controller waiting forever.
cancelCommittedTimer();
const owner = generation;
armFallbackTimer(function () {
return awaitingCommit && generation === owner;
});
}
function onCompositionEnd(event) {
if (finalizedByKeydown) {
finalizedByKeydown = false;
return;
}
finalizeComposition(event.data, false);
}
// Mirrors CompositionHelper.keydown in @xterm/xterm 6.0.0
// (src/browser/input/CompositionHelper.ts:94-108): while composing, keyCode
// 20/229 and 16/17/18 keep the composition open and every other keyCode
// finalizes it. `isComposing` and `key` are deliberately not consulted,
// because xterm does not consult them.
function onKeydown(event) {
if (keydownTarget !== textarea && event.target !== textarea) return;
if (!composing || KEEP_COMPOSING_KEYCODES.has(event.keyCode)) return;
finalizeComposition(latestValue, true);
}
function reset() {
if (destroyed) return;
cleanup();
}
function consumeTerminalData(data) {
if (
destroyed ||
!awaitingCommit ||
typeof data !== 'string' ||
data.length === 0 ||
CONTROL_OR_LINE_BREAK.test(data)
) {
return false;
}
generation += 1;
const owner = generation;
cancelScheduledFrame();
cancelCommittedTimer();
composing = false;
awaitingCommit = false;
committed = true;
latestValue = data;
renderPhase = 'committed';
safely(onCommit, data);
if (destroyed || generation !== owner || !committed) return true;
scheduleLatestPreview('committed');
if (destroyed || generation !== owner || !committed) return true;
const armed = armFallbackTimer(function () {
return committed;
});
if (!armed && !destroyed && generation === owner && committed) cleanup();
return true;
}
function completeCommit(result) {
if (destroyed || !result || result.predicted !== true || !committed) return;
cleanup();
}
function noteAuthoritativeOutput() {
if (destroyed || !committed) return;
cleanup();
}
const listeners = [
[textarea, 'compositionstart', beginComposition],
[textarea, 'compositionupdate', updateComposition],
[textarea, 'input', onComposingInput],
[textarea, 'compositionend', onCompositionEnd],
[keydownTarget, 'keydown', onKeydown, true],
[textarea, 'blur', reset],
];
for (const [target, type, listener, capture] of listeners) target.addEventListener(type, listener, capture);
function destroy() {
if (destroyed) return;
destroyed = true;
for (const [target, type, listener, capture] of listeners) target.removeEventListener(type, listener, capture);
cleanup();
}
return {
consumeTerminalData,
completeCommit,
noteAuthoritativeOutput,
reset,
destroy,
get state() {
return {
generation,
composing,
awaitingCommit,
committed,
latest: latestValue,
framePending: frameToken !== null,
timerPending: timerToken !== null,
};
},
};
}
global.MobileImePreview = { create, isIosWebKitTouch };
})(globalThis);
+25
View File
@@ -460,6 +460,31 @@ textarea:focus-visible {
font-size: 16px !important; /* prevent iOS auto-zoom on focus */
}
.xterm-helpers .codeman-ime-preview {
position: absolute;
left: var(--xterm-helper-left, 0px);
top: var(--xterm-helper-top, 0px);
z-index: 6;
pointer-events: none;
white-space: pre;
color: var(--terminal-foreground, var(--text, #fff));
font-family: var(--font-mono, monospace);
font-size: 14px;
font-weight: 400;
font-style: normal;
line-height: 1.2;
height: 1.2em;
}
.xterm-helpers .codeman-ime-preview[data-phase='provisional'] {
text-decoration: underline;
text-decoration-style: dotted;
}
.touch-device .xterm-helpers.codeman-ime-preview-owned .composition-view.active {
display: none;
}
/* Session tab focus */
.session-tab:focus-visible {
outline: 2px solid var(--accent);
+254
View File
@@ -250,6 +250,235 @@ Object.assign(CodemanApp.prototype, {
this._keyCode229Recovery = null;
},
_destroyMobileImePreview() {
try {
this._mobileImePreview?.destroy?.();
} catch {
// The preview is visual-only; terminal replacement must continue.
}
this._mobileImePreview = null;
this._mobileImePreviewSessionId = null;
this._mobileImeCommitOutputSeq = null;
try {
this._mobileImePreviewNode?.remove?.();
} catch {
// Best-effort node cleanup only.
}
try {
this._mobileImePreviewHelpers?.classList?.remove('codeman-ime-preview-owned');
} catch {
// Best-effort ownership cleanup only.
}
this._mobileImePreviewNode = null;
this._mobileImePreviewHelpers = null;
try {
if (this._mobileImePreviewOfflineHandler) {
window.removeEventListener('offline', this._mobileImePreviewOfflineHandler);
}
if (this._mobileImePreviewPagehideHandler) {
window.removeEventListener('pagehide', this._mobileImePreviewPagehideHandler);
}
} catch {
// Best-effort listener cleanup only.
}
this._mobileImePreviewOfflineHandler = null;
this._mobileImePreviewPagehideHandler = null;
},
/**
* iOS Safari IME preview (mobile-ime-preview.js). WebKit does not show the
* text an IME is composing inside the terminal, so the user types blind.
*
* Two homes, chosen per render:
* - Local echo on: typed text sits in the LocalEchoOverlay and the PTY
* cursor stays at the prompt start, under the overlay's opaque text (z 7,
* `.xterm-screen`). So the overlay draws the composition itself, as an
* underlined tail after its pending text (`setComposition`).
* - Otherwise (a shell, or the overlay could not place it): a span inside
* `.xterm-helpers`, positioned by the same --xterm-helper-left/top vars as
* the helper textarea, which follow the PTY cursor.
*
* Visual only: nothing here touches the input path, and every failure
* leaves no DOM behind.
*/
_initMobileImePreview() {
this._destroyMobileImePreview();
let preview = null;
let helpers = null;
try {
if (typeof MobileImePreview === 'undefined' || !MobileImePreview?.isIosWebKitTouch?.()) return;
const textarea = this.terminal?.textarea;
helpers = this.terminal?.element?.querySelector?.('.xterm-helpers');
if (!textarea || !helpers) return;
preview = document.createElement('span');
this._mobileImePreviewNode = preview;
this._mobileImePreviewHelpers = helpers;
preview.className = 'codeman-ime-preview';
preview.setAttribute('aria-hidden', 'true');
preview.hidden = true;
helpers.appendChild(preview);
const syncPreviewTypography = () => {
try {
const compositionView =
helpers.querySelector?.('.composition-view') || this.terminal?.element?.querySelector?.('.composition-view');
if (!compositionView || !preview.style) return;
const style = typeof getComputedStyle === 'function' ? getComputedStyle(compositionView) : compositionView.style;
for (const property of ['fontFamily', 'fontSize', 'fontWeight', 'fontStyle', 'lineHeight', 'height']) {
const value = style?.[property] || compositionView.style?.[property];
if (value) preview.style[property] = value;
}
const theme = this.terminal?.options?.theme;
let foreground = theme?.foreground;
let background = theme?.background;
if (!foreground || !background) {
try {
const current = window.codemanCurrentXtermTheme?.();
foreground = foreground || current?.foreground;
background = background || current?.background;
} catch {
// Theme lookup is best-effort; retain the safe terminal fallback.
}
}
preview.style.color = foreground || '#e0e0e0';
// Opaque, like xterm's own composition view, so the preview does not
// overprint whatever sits at the cursor (a dim composer placeholder).
preview.style.backgroundColor = background || '#0d0d0d';
} catch {
// Typography matching is visual-only and must not block input.
}
};
// The overlay only when it is what shows typed text right now (local echo
// on, and not handed back to plain PTY echo by a composer nav key).
const localEchoOverlay = () =>
this._localEchoEnabled && !this._echoPassthroughSessions?.has(this.activeSessionId)
? this._localEchoOverlay || null
: null;
const clearOverlayComposition = () => {
try {
if (this._localEchoOverlay?.composition) this._localEchoOverlay.setComposition('');
} catch {}
};
const hideSpan = () => {
try {
preview.hidden = true;
} catch {}
try {
preview.textContent = '';
} catch {}
try {
delete preview.dataset.phase;
} catch {}
try {
helpers.classList.remove('codeman-ime-preview-owned');
} catch {}
};
const clearPreview = () => {
clearOverlayComposition();
hideSpan();
};
const controller = MobileImePreview.create({
textarea,
// An ancestor of the textarea: its capture-phase keydown listener runs
// before xterm's capture listener on the textarea, which finalizes the
// composition and emits the commit synchronously.
keydownTarget: this.terminal.element,
render: ({ text, phase }) => {
try {
const overlay = localEchoOverlay();
if (overlay && typeof overlay.setComposition === 'function') {
overlay.setComposition(text);
// No prompt found = nothing drawn: fall back to the span.
if (!text || overlay.state?.visible) {
hideSpan();
helpers.classList.toggle('codeman-ime-preview-owned', !!text);
return;
}
overlay.setComposition('');
} else {
clearOverlayComposition();
}
syncPreviewTypography();
preview.textContent = text;
preview.dataset.phase = phase;
preview.hidden = !text;
helpers.classList.toggle('codeman-ime-preview-owned', !!text);
} catch {
clearPreview();
}
},
clear: clearPreview,
});
this._mobileImePreview = controller;
this._mobileImePreviewSessionId = this.activeSessionId;
// Offline and pagehide only reset: initTerminal() runs once per page
// load, so destroying on pagehide would leave the preview off for good
// after a back-forward cache restore (iOS Safari keeps pages there).
this._mobileImePreviewOfflineHandler = () => {
try {
this._mobileImePreview?.reset?.();
} catch {
// Disconnect cleanup is visual-only.
}
};
this._mobileImePreviewPagehideHandler = this._mobileImePreviewOfflineHandler;
window.addEventListener('offline', this._mobileImePreviewOfflineHandler);
window.addEventListener('pagehide', this._mobileImePreviewPagehideHandler);
} catch {
this._destroyMobileImePreview();
}
},
/**
* Tell the IME preview about a chunk xterm emitted through onData. Returns
* true when the chunk is the IME's committed text, in which case the preview
* holds it (phase 'committed') until something else shows it. Never throws.
*/
_consumeMobileImeTerminalData(data) {
let isImeCommit = false;
try {
isImeCommit = this._mobileImePreview?.consumeTerminalData?.(data) === true;
} catch {
// The preview is visual-only; normal terminal input must continue.
}
// Output accepted from here on can carry the echo of this commit.
if (isImeCommit) this._mobileImeCommitOutputSeq = this._terminalOutputSeq || 0;
return isImeCommit;
},
/**
* Clear a committed IME preview once terminal output accepted AFTER the
* commit has been parsed. `flushedOutputSeq` is the output sequence a fully
* written flush covered (null when part of it was deferred), so output that
* was already queued before the commit can never clear it early.
*/
_noteMobileImeAuthoritativeOutput(flushedOutputSeq, sessionId) {
try {
const commitSeq = this._mobileImeCommitOutputSeq;
if (commitSeq === null || commitSeq === undefined || flushedOutputSeq === null) return;
if (sessionId !== this.activeSessionId || !(flushedOutputSeq > commitSeq)) return;
this._mobileImeCommitOutputSeq = null;
this._mobileImePreview?.noteAuthoritativeOutput?.();
} catch {
// Authoritative output is never delayed or consumed by the preview.
}
},
/**
* The local echo overlay has just taken a committed IME chunk through the
* ordinary printable/paste branch, so it now shows the text: release the
* preview instead of waiting for terminal output.
*/
_transferMobileImeCommitToLocalEcho() {
this._mobileImeCommitOutputSeq = null;
try {
this._mobileImePreview?.completeCommit?.({ predicted: true });
} catch {
// Ownership transfer is visual-only.
}
},
initTerminal() {
// Load scrollback setting from localStorage, treating DEFAULT_SCROLLBACK as a floor
// so users who picked up the previous (smaller) default get the new minimum on upgrade.
@@ -307,6 +536,7 @@ Object.assign(CodemanApp.prototype, {
const container = document.getElementById('terminalContainer');
this.terminal.open(container);
this._initMobileImePreview();
this._installMobileTapMouseGuard();
this._installShiftDragSelection();
this._installTouchSelectionFocusGuard();
@@ -1215,6 +1445,8 @@ Object.assign(CodemanApp.prototype, {
// survives tab switches and reconnects.
const handleTerminalData = (data) => {
// Before anything can rewrite `data`: is this chunk the IME's commit?
const isImeCommit = this._consumeMobileImeTerminalData(data);
// Mouse SGR reports (tap-to-position) are NOT IME input — they must reach
// the PTY even while the CJK input field owns focus. Without this exception
// tapping to move the cursor silently does nothing whenever Chinese input
@@ -1292,6 +1524,8 @@ Object.assign(CodemanApp.prototype, {
// When enabled, keystrokes are buffered locally in the overlay for
// instant visual feedback. Nothing is sent to the PTY until Enter
// (or a control char) is pressed — avoids out-of-order char delivery.
// An IME commit takes the same printable/paste branch as typed text,
// and the overlay then shows it in place of the preview.
if (this._localEchoEnabled && !echoPassthrough) {
if (data === '\x7f') {
const source = this._localEchoOverlay?.removeChar();
@@ -1348,6 +1582,7 @@ Object.assign(CodemanApp.prototype, {
if (data.length > 1 && data.charCodeAt(0) >= 32) {
// Paste: append to overlay only (sent on Enter)
this._localEchoOverlay?.appendText(data);
if (isImeCommit && this._localEchoOverlay) this._transferMobileImeCommitToLocalEcho();
return;
}
if (data.charCodeAt(0) < 32) {
@@ -1493,6 +1728,7 @@ Object.assign(CodemanApp.prototype, {
if (data.length === 1 && data.charCodeAt(0) >= 32) {
// Printable char: add to overlay only (sent on Enter)
this._localEchoOverlay?.addChar(data);
if (isImeCommit && this._localEchoOverlay) this._transferMobileImeCommitToLocalEcho();
return;
}
}
@@ -3528,6 +3764,9 @@ Object.assign(CodemanApp.prototype, {
},
batchTerminalWrite(data) {
// Arrival order of output, so the IME preview can tell output that
// followed a commit from output that was already queued before it.
this._terminalOutputSeq = (this._terminalOutputSeq || 0) + 1;
// Feed the renderer watchdog. Recorded before the buffer-load early return
// below: a write that is queued rather than written still means the pipeline
// owes us a frame once it drains.
@@ -3591,6 +3830,7 @@ Object.assign(CodemanApp.prototype, {
// Accumulate raw data (may contain DEC 2026 markers)
this.pendingWrites.push(data);
this._pendingWritesOutputSeq = this._terminalOutputSeq;
this._scheduleTerminalWriteFlush();
},
@@ -3622,6 +3862,7 @@ Object.assign(CodemanApp.prototype, {
// Transfer buffered data to normal pending writes
this.pendingWrites.push(this.flickerFilterBuffer);
this._pendingWritesOutputSeq = this._terminalOutputSeq;
this.flickerFilterBuffer = '';
this.flickerFilterActive = false;
@@ -3652,6 +3893,15 @@ Object.assign(CodemanApp.prototype, {
* Position is tracked dynamically by _findPrompt() on every render.
*/
_updateLocalEchoState() {
if (this._mobileImePreviewSessionId !== this.activeSessionId) {
this._mobileImePreviewSessionId = this.activeSessionId;
this._mobileImeCommitOutputSeq = null;
try {
this._mobileImePreview?.reset?.();
} catch {
// The preview is visual-only; session switching must continue.
}
}
const settings = this.loadAppSettingsFromStorage();
const session = this.activeSessionId ? this.sessions.get(this.activeSessionId) : null;
const echoEnabled = settings.localEchoEnabled ?? MobileDetection.isTouchDevice();
@@ -3857,6 +4107,9 @@ Object.assign(CodemanApp.prototype, {
this.pendingWrites.push(joined.slice(MAX_FRAME_BYTES));
deferred = true;
}
// Newest output this chunk fully contains, for the IME preview. A split
// chunk may not hold that output yet, so it reports nothing.
const flushedOutputSeq = deferred ? null : (this._pendingWritesOutputSeq ?? null);
this._terminalWriteInFlight = true;
this._terminalWriteInFlightBytes = writeChunk.length;
try {
@@ -3874,6 +4127,7 @@ Object.assign(CodemanApp.prototype, {
// because the test's write mock moved the viewport synchronously.)
this._restoreTerminalViewport(preserveViewportY, flushSessionId);
this._scheduleTerminalWriteFlush();
this._noteMobileImeAuthoritativeOutput(flushedOutputSeq, flushSessionId);
});
} catch (err) {
this._terminalWriteInFlight = false;