feat(web): mobile image upload to active session via paste dialog (#101)

- Extend the keyboard accessory paste dialog with an image picker
  (camera / photo library) plus best-effort image paste, routing
  selected files through the existing _uploadAndInsertImages pipeline
- Re-encode images to standard JPEG/PNG in the browser before upload,
  so mislabeled gallery images (e.g. MIUI WebP claiming image/jpeg)
  pass the server magic-byte check; PNG keeps transparency, GIF passes
  through untouched, decode failures fall back to the original file
- Log the real byte header on the paste-image magic-mismatch branch to
  pin down any remaining format mismatches without a guessing loop
- Ignore the runtime .claude-images/ upload directory

Co-authored-by: Teigen <teigen@TeigendeMac-mini.local>
This commit is contained in:
Tenggan Zhang
2026-06-01 19:38:48 +02:00
committed by GitHub
co-authored by Teigen
parent 06f9ff6d9c
commit ec38fd11bf
5 changed files with 130 additions and 10 deletions
+3
View File
@@ -66,3 +66,6 @@ commands
todo.md
@fix_plan.md
readme-preview.mjs
# Uploaded images land here under each session working dir (runtime artifact)
.claude-images/
+53 -1
View File
@@ -113,7 +113,14 @@ Object.assign(CodemanApp.prototype, {
const paths = [];
for (const file of files) {
try {
const path = await this._uploadPasteImage(sessionId, file);
// Re-encode to a standard JPEG/PNG before upload. Galleries on some
// phones (notably Android/MIUI) hand back a WebP/HEIF whose filename and
// MIME claim "image/jpeg", which passes the server's extension allowlist
// but fails its magic-byte check ("bytes do not match declared type").
// Decoding through the browser and re-encoding guarantees the bytes
// match the extension we send.
const normalized = await this._normalizeImageForUpload(file);
const path = await this._uploadPasteImage(sessionId, normalized);
paths.push(path);
} catch (err) {
this.showToast('Upload failed: ' + (err.message || 'unknown error'), 'error');
@@ -145,4 +152,49 @@ Object.assign(CodemanApp.prototype, {
return data.path;
},
// Decode an image File through the browser and re-encode it to a format the
// server accepts, so the uploaded bytes always match their declared
// extension. PNG is re-encoded as PNG (preserves transparency); everything
// else (JPEG, WebP, HEIF, unknown) becomes JPEG. Animated GIFs are passed
// through untouched since a canvas would flatten them to one frame. On any
// decode/encode failure the original file is returned unchanged so the server
// still gets a chance (and logs a precise diagnostic).
async _normalizeImageForUpload(file) {
if (file.type === 'image/gif') return file;
const toPng = file.type === 'image/png';
const url = URL.createObjectURL(file);
try {
const img = new Image();
await new Promise((resolve, reject) => {
img.onload = () => resolve();
img.onerror = () => reject(new Error('decode failed'));
img.src = url;
});
const width = img.naturalWidth;
const height = img.naturalHeight;
if (!width || !height) return file;
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
if (!ctx) return file;
ctx.drawImage(img, 0, 0);
const mime = toPng ? 'image/png' : 'image/jpeg';
const blob = await new Promise((resolve) => canvas.toBlob(resolve, mime, 0.92));
if (!blob) return file;
const baseName = (file.name || 'image').replace(/\.[^.]+$/, '') || 'image';
return new File([blob], baseName + (toPng ? '.png' : '.jpg'), { type: mime });
} catch (err) {
console.warn('Image re-encode failed, uploading original:', err);
return file;
} finally {
URL.revokeObjectURL(url);
}
},
});
+57 -8
View File
@@ -5,6 +5,8 @@
*
* - KeyboardAccessoryBar (singleton object) — Quick action buttons shown above the virtual
* keyboard on mobile: arrow up/down, /init, /clear, /compact, paste, and dismiss.
* The paste button opens a dialog that handles both text paste and image attach
* (native picker + best-effort image paste, routed through app._uploadAndInsertImages).
* Destructive actions (/clear, /compact) require double-tap confirmation (2s amber state).
* Commands are sent as text + Enter separately for Ink compatibility.
* Only initializes on touch devices (MobileDetection.isTouchDevice guard).
@@ -264,8 +266,17 @@ const KeyboardAccessoryBar = {
}).catch(() => {});
},
/** Read clipboard and send contents as input */
/** Show a paste overlay with a textarea for iOS compatibility */
/** Show a paste overlay for iOS compatibility.
* Handles three input paths from one dialog:
* - Text: long-press the textarea → Paste → Send (unchanged).
* - Image (picker): the "Image" button opens a native file picker
* (accept=image/* → camera / photo library / files), the most reliable
* way to attach a photo on mobile.
* - Image (paste): if the browser exposes image blobs on the textarea's
* paste event, we intercept them and upload directly. Support is spotty
* on mobile, so it is a best-effort enhancement layered on the picker.
* All image paths reuse app._uploadAndInsertImages() (image-input.js), which
* uploads to /api/sessions/:id/paste-image and inserts the saved path. */
pasteFromClipboard() {
if (typeof app === 'undefined' || !app.activeSessionId) return;
@@ -274,23 +285,61 @@ const KeyboardAccessoryBar = {
overlay.className = 'paste-overlay';
overlay.innerHTML = `
<div class="paste-dialog">
<textarea class="paste-textarea" placeholder="Long-press here and tap Paste"></textarea>
<textarea class="paste-textarea" placeholder="Long-press to paste text — or tap 🖼 to attach an image"></textarea>
<div class="paste-actions">
<button class="paste-image">🖼 Image</button>
<button class="paste-cancel">Cancel</button>
<button class="paste-send">Send</button>
</div>
<input type="file" class="paste-file-input" accept="image/*" multiple hidden>
</div>
`;
const textarea = overlay.querySelector('.paste-textarea');
const send = () => {
const fileInput = overlay.querySelector('.paste-file-input');
const close = () => overlay.remove();
const sendText = () => {
const text = textarea.value;
overlay.remove();
close();
if (text) app.sendInput(text);
};
overlay.querySelector('.paste-cancel').addEventListener('click', () => overlay.remove());
overlay.querySelector('.paste-send').addEventListener('click', send);
overlay.addEventListener('click', (e) => { if (e.target === overlay) overlay.remove(); });
// Filter to images, close the dialog, and hand off to the shared
// upload+insert pipeline. Returns true if any image was handled.
const handleImages = (files) => {
const images = Array.from(files || []).filter((f) => f.type.startsWith('image/'));
if (images.length === 0) return false;
close();
if (typeof app._uploadAndInsertImages === 'function') app._uploadAndInsertImages(images);
return true;
};
// Image picker (camera / photo library) — the reliable mobile path.
overlay.querySelector('.paste-image').addEventListener('click', () => fileInput.click());
fileInput.addEventListener('change', () => handleImages(fileInput.files));
// Best-effort: capture images pasted straight into the textarea.
textarea.addEventListener('paste', (e) => {
const items = e.clipboardData && e.clipboardData.items;
if (!items) return;
const imageFiles = [];
for (let i = 0; i < items.length; i++) {
if (items[i].type.startsWith('image/')) {
const blob = items[i].getAsFile();
if (blob) imageFiles.push(blob);
}
}
if (imageFiles.length > 0) {
e.preventDefault();
handleImages(imageFiles);
}
});
overlay.querySelector('.paste-cancel').addEventListener('click', close);
overlay.querySelector('.paste-send').addEventListener('click', sendText);
overlay.addEventListener('click', (e) => { if (e.target === overlay) close(); });
document.body.appendChild(overlay);
textarea.focus();
+9 -1
View File
@@ -1043,7 +1043,7 @@ html.mobile-init .file-browser-panel {
margin-top: 10px;
}
.paste-cancel, .paste-new, .paste-send {
.paste-cancel, .paste-new, .paste-send, .paste-image {
padding: 8px 18px;
border: none;
border-radius: 8px;
@@ -1051,6 +1051,14 @@ html.mobile-init .file-browser-panel {
cursor: pointer;
}
/* Image attach button — left-aligned, accent outline */
.paste-image {
margin-right: auto;
background: var(--bg-tertiary, #333);
color: var(--accent-color, #7aa2f7);
border: 1px solid var(--accent-color, #7aa2f7);
}
.paste-cancel {
background: var(--bg-tertiary, #333);
color: var(--text-secondary, #aaa);
+8
View File
@@ -1674,6 +1674,14 @@ export function registerSessionRoutes(
// Sniff actual bytes — filename and Content-Type are both attacker-supplied.
// Polyglot HTML/PNG would otherwise pass and serve back with image/png MIME.
if (!imageMagicMatchesExt(imageBytes, ext)) {
// Diagnostic: on some Android galleries (e.g. MIUI) a WebP/HEIF is
// mislabeled as image/jpeg, so the declared ext passes the allowlist but
// the magic bytes do not. Log the real header so format mismatches can be
// pinned down without a reproduce-and-guess loop. The client now
// re-encodes images to JPEG/PNG before upload, so this should be rare.
console.warn(
`[paste-image] magic mismatch: filename=${JSON.stringify(part.filename)} mime=${JSON.stringify(part.mimetype)} declaredExt=${ext} magic=${imageBytes.subarray(0, 12).toString('hex')}`
);
reply.code(415);
return createErrorResponse(ApiErrorCode.INVALID_INPUT, `Image bytes do not match declared type ${ext}`);
}