mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
feat(web): mobile image upload to active session via paste dialog (#101)
- Extend the keyboard accessory paste dialog with an image picker (camera / photo library) plus best-effort image paste, routing selected files through the existing _uploadAndInsertImages pipeline - Re-encode images to standard JPEG/PNG in the browser before upload, so mislabeled gallery images (e.g. MIUI WebP claiming image/jpeg) pass the server magic-byte check; PNG keeps transparency, GIF passes through untouched, decode failures fall back to the original file - Log the real byte header on the paste-image magic-mismatch branch to pin down any remaining format mismatches without a guessing loop - Ignore the runtime .claude-images/ upload directory Co-authored-by: Teigen <teigen@TeigendeMac-mini.local>
This commit is contained in:
@@ -66,3 +66,6 @@ commands
|
||||
todo.md
|
||||
@fix_plan.md
|
||||
readme-preview.mjs
|
||||
|
||||
# Uploaded images land here under each session working dir (runtime artifact)
|
||||
.claude-images/
|
||||
|
||||
@@ -113,7 +113,14 @@ Object.assign(CodemanApp.prototype, {
|
||||
const paths = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const path = await this._uploadPasteImage(sessionId, file);
|
||||
// Re-encode to a standard JPEG/PNG before upload. Galleries on some
|
||||
// phones (notably Android/MIUI) hand back a WebP/HEIF whose filename and
|
||||
// MIME claim "image/jpeg", which passes the server's extension allowlist
|
||||
// but fails its magic-byte check ("bytes do not match declared type").
|
||||
// Decoding through the browser and re-encoding guarantees the bytes
|
||||
// match the extension we send.
|
||||
const normalized = await this._normalizeImageForUpload(file);
|
||||
const path = await this._uploadPasteImage(sessionId, normalized);
|
||||
paths.push(path);
|
||||
} catch (err) {
|
||||
this.showToast('Upload failed: ' + (err.message || 'unknown error'), 'error');
|
||||
@@ -145,4 +152,49 @@ Object.assign(CodemanApp.prototype, {
|
||||
return data.path;
|
||||
},
|
||||
|
||||
// Decode an image File through the browser and re-encode it to a format the
|
||||
// server accepts, so the uploaded bytes always match their declared
|
||||
// extension. PNG is re-encoded as PNG (preserves transparency); everything
|
||||
// else (JPEG, WebP, HEIF, unknown) becomes JPEG. Animated GIFs are passed
|
||||
// through untouched since a canvas would flatten them to one frame. On any
|
||||
// decode/encode failure the original file is returned unchanged so the server
|
||||
// still gets a chance (and logs a precise diagnostic).
|
||||
async _normalizeImageForUpload(file) {
|
||||
if (file.type === 'image/gif') return file;
|
||||
|
||||
const toPng = file.type === 'image/png';
|
||||
const url = URL.createObjectURL(file);
|
||||
try {
|
||||
const img = new Image();
|
||||
await new Promise((resolve, reject) => {
|
||||
img.onload = () => resolve();
|
||||
img.onerror = () => reject(new Error('decode failed'));
|
||||
img.src = url;
|
||||
});
|
||||
|
||||
const width = img.naturalWidth;
|
||||
const height = img.naturalHeight;
|
||||
if (!width || !height) return file;
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return file;
|
||||
ctx.drawImage(img, 0, 0);
|
||||
|
||||
const mime = toPng ? 'image/png' : 'image/jpeg';
|
||||
const blob = await new Promise((resolve) => canvas.toBlob(resolve, mime, 0.92));
|
||||
if (!blob) return file;
|
||||
|
||||
const baseName = (file.name || 'image').replace(/\.[^.]+$/, '') || 'image';
|
||||
return new File([blob], baseName + (toPng ? '.png' : '.jpg'), { type: mime });
|
||||
} catch (err) {
|
||||
console.warn('Image re-encode failed, uploading original:', err);
|
||||
return file;
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
},
|
||||
|
||||
});
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
*
|
||||
* - KeyboardAccessoryBar (singleton object) — Quick action buttons shown above the virtual
|
||||
* keyboard on mobile: arrow up/down, /init, /clear, /compact, paste, and dismiss.
|
||||
* The paste button opens a dialog that handles both text paste and image attach
|
||||
* (native picker + best-effort image paste, routed through app._uploadAndInsertImages).
|
||||
* Destructive actions (/clear, /compact) require double-tap confirmation (2s amber state).
|
||||
* Commands are sent as text + Enter separately for Ink compatibility.
|
||||
* Only initializes on touch devices (MobileDetection.isTouchDevice guard).
|
||||
@@ -264,8 +266,17 @@ const KeyboardAccessoryBar = {
|
||||
}).catch(() => {});
|
||||
},
|
||||
|
||||
/** Read clipboard and send contents as input */
|
||||
/** Show a paste overlay with a textarea for iOS compatibility */
|
||||
/** Show a paste overlay for iOS compatibility.
|
||||
* Handles three input paths from one dialog:
|
||||
* - Text: long-press the textarea → Paste → Send (unchanged).
|
||||
* - Image (picker): the "Image" button opens a native file picker
|
||||
* (accept=image/* → camera / photo library / files), the most reliable
|
||||
* way to attach a photo on mobile.
|
||||
* - Image (paste): if the browser exposes image blobs on the textarea's
|
||||
* paste event, we intercept them and upload directly. Support is spotty
|
||||
* on mobile, so it is a best-effort enhancement layered on the picker.
|
||||
* All image paths reuse app._uploadAndInsertImages() (image-input.js), which
|
||||
* uploads to /api/sessions/:id/paste-image and inserts the saved path. */
|
||||
pasteFromClipboard() {
|
||||
if (typeof app === 'undefined' || !app.activeSessionId) return;
|
||||
|
||||
@@ -274,23 +285,61 @@ const KeyboardAccessoryBar = {
|
||||
overlay.className = 'paste-overlay';
|
||||
overlay.innerHTML = `
|
||||
<div class="paste-dialog">
|
||||
<textarea class="paste-textarea" placeholder="Long-press here and tap Paste"></textarea>
|
||||
<textarea class="paste-textarea" placeholder="Long-press to paste text — or tap 🖼 to attach an image"></textarea>
|
||||
<div class="paste-actions">
|
||||
<button class="paste-image">🖼 Image</button>
|
||||
<button class="paste-cancel">Cancel</button>
|
||||
<button class="paste-send">Send</button>
|
||||
</div>
|
||||
<input type="file" class="paste-file-input" accept="image/*" multiple hidden>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const textarea = overlay.querySelector('.paste-textarea');
|
||||
const send = () => {
|
||||
const fileInput = overlay.querySelector('.paste-file-input');
|
||||
|
||||
const close = () => overlay.remove();
|
||||
|
||||
const sendText = () => {
|
||||
const text = textarea.value;
|
||||
overlay.remove();
|
||||
close();
|
||||
if (text) app.sendInput(text);
|
||||
};
|
||||
overlay.querySelector('.paste-cancel').addEventListener('click', () => overlay.remove());
|
||||
overlay.querySelector('.paste-send').addEventListener('click', send);
|
||||
overlay.addEventListener('click', (e) => { if (e.target === overlay) overlay.remove(); });
|
||||
|
||||
// Filter to images, close the dialog, and hand off to the shared
|
||||
// upload+insert pipeline. Returns true if any image was handled.
|
||||
const handleImages = (files) => {
|
||||
const images = Array.from(files || []).filter((f) => f.type.startsWith('image/'));
|
||||
if (images.length === 0) return false;
|
||||
close();
|
||||
if (typeof app._uploadAndInsertImages === 'function') app._uploadAndInsertImages(images);
|
||||
return true;
|
||||
};
|
||||
|
||||
// Image picker (camera / photo library) — the reliable mobile path.
|
||||
overlay.querySelector('.paste-image').addEventListener('click', () => fileInput.click());
|
||||
fileInput.addEventListener('change', () => handleImages(fileInput.files));
|
||||
|
||||
// Best-effort: capture images pasted straight into the textarea.
|
||||
textarea.addEventListener('paste', (e) => {
|
||||
const items = e.clipboardData && e.clipboardData.items;
|
||||
if (!items) return;
|
||||
const imageFiles = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
if (items[i].type.startsWith('image/')) {
|
||||
const blob = items[i].getAsFile();
|
||||
if (blob) imageFiles.push(blob);
|
||||
}
|
||||
}
|
||||
if (imageFiles.length > 0) {
|
||||
e.preventDefault();
|
||||
handleImages(imageFiles);
|
||||
}
|
||||
});
|
||||
|
||||
overlay.querySelector('.paste-cancel').addEventListener('click', close);
|
||||
overlay.querySelector('.paste-send').addEventListener('click', sendText);
|
||||
overlay.addEventListener('click', (e) => { if (e.target === overlay) close(); });
|
||||
|
||||
document.body.appendChild(overlay);
|
||||
textarea.focus();
|
||||
|
||||
@@ -1043,7 +1043,7 @@ html.mobile-init .file-browser-panel {
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.paste-cancel, .paste-new, .paste-send {
|
||||
.paste-cancel, .paste-new, .paste-send, .paste-image {
|
||||
padding: 8px 18px;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
@@ -1051,6 +1051,14 @@ html.mobile-init .file-browser-panel {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* Image attach button — left-aligned, accent outline */
|
||||
.paste-image {
|
||||
margin-right: auto;
|
||||
background: var(--bg-tertiary, #333);
|
||||
color: var(--accent-color, #7aa2f7);
|
||||
border: 1px solid var(--accent-color, #7aa2f7);
|
||||
}
|
||||
|
||||
.paste-cancel {
|
||||
background: var(--bg-tertiary, #333);
|
||||
color: var(--text-secondary, #aaa);
|
||||
|
||||
@@ -1674,6 +1674,14 @@ export function registerSessionRoutes(
|
||||
// Sniff actual bytes — filename and Content-Type are both attacker-supplied.
|
||||
// Polyglot HTML/PNG would otherwise pass and serve back with image/png MIME.
|
||||
if (!imageMagicMatchesExt(imageBytes, ext)) {
|
||||
// Diagnostic: on some Android galleries (e.g. MIUI) a WebP/HEIF is
|
||||
// mislabeled as image/jpeg, so the declared ext passes the allowlist but
|
||||
// the magic bytes do not. Log the real header so format mismatches can be
|
||||
// pinned down without a reproduce-and-guess loop. The client now
|
||||
// re-encodes images to JPEG/PNG before upload, so this should be rare.
|
||||
console.warn(
|
||||
`[paste-image] magic mismatch: filename=${JSON.stringify(part.filename)} mime=${JSON.stringify(part.mimetype)} declaredExt=${ext} magic=${imageBytes.subarray(0, 12).toString('hex')}`
|
||||
);
|
||||
reply.code(415);
|
||||
return createErrorResponse(ApiErrorCode.INVALID_INPUT, `Image bytes do not match declared type ${ext}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user