diff --git a/crates/app-tauri/src/chat.rs b/crates/app-tauri/src/chat.rs index 84aa5252..b5e866f3 100644 --- a/crates/app-tauri/src/chat.rs +++ b/crates/app-tauri/src/chat.rs @@ -145,7 +145,9 @@ impl ChatBridge { fn reply_chunk_bytes(chunk: &ReplyChunk) -> usize { match chunk { - ReplyChunk::UserPrompt { text } => text.len(), + ReplyChunk::UserPrompt { text, attachments } => { + text.len() + attachments.iter().map(String::len).sum::() + } ReplyChunk::TextDelta { text } => text.len(), ReplyChunk::ToolActivity { label } => label.len(), ReplyChunk::Progress { progress } => { diff --git a/crates/app-tauri/src/commands.rs b/crates/app-tauri/src/commands.rs index 5d2c49d9..b4089e46 100644 --- a/crates/app-tauri/src/commands.rs +++ b/crates/app-tauri/src/commands.rs @@ -2356,6 +2356,10 @@ pub async fn agent_send( ) .await?; let prompt_for_model = prompt_with_imported_attachments(&prompt, &imported_attachments); + let attachment_labels = imported_attachments + .iter() + .map(|attachment| attachment.filename.clone()) + .collect(); // Retain the human submit in the same live scrollback as model chunks so // `reattach_agent_chat` can repaint the whole in-flight conversation. @@ -2363,6 +2367,7 @@ pub async fn agent_send( &sid, ReplyChunk::UserPrompt { text: prompt.clone(), + attachments: attachment_labels, }, ); diff --git a/crates/app-tauri/tests/chat_bridge.rs b/crates/app-tauri/tests/chat_bridge.rs index 685bfed9..112b4df1 100644 --- a/crates/app-tauri/tests/chat_bridge.rs +++ b/crates/app-tauri/tests/chat_bridge.rs @@ -61,7 +61,9 @@ fn error_chunk(s: &str) -> ReplyChunk { fn chunk_bytes(chunk: &ReplyChunk) -> usize { match chunk { - ReplyChunk::UserPrompt { text } => text.len(), + ReplyChunk::UserPrompt { text, attachments } => { + text.len() + attachments.iter().map(String::len).sum::() + } ReplyChunk::TextDelta { text } => text.len(), ReplyChunk::ToolActivity { label } => label.len(), ReplyChunk::Progress { progress } => { diff --git a/crates/app-tauri/tests/dto_chat.rs b/crates/app-tauri/tests/dto_chat.rs index c88da3b7..d8e0b051 100644 --- a/crates/app-tauri/tests/dto_chat.rs +++ b/crates/app-tauri/tests/dto_chat.rs @@ -27,11 +27,29 @@ use uuid::Uuid; fn reply_chunk_user_prompt_serialises_exact_camel_case() { let v = serde_json::to_value(ReplyChunk::UserPrompt { text: "run tests".into(), + attachments: Vec::new(), }) .unwrap(); assert_eq!(v, json!({ "kind": "userPrompt", "text": "run tests" })); } +#[test] +fn reply_chunk_user_prompt_serialises_attachment_labels() { + let v = serde_json::to_value(ReplyChunk::UserPrompt { + text: "describe this".into(), + attachments: vec!["clipboard.png".into()], + }) + .unwrap(); + assert_eq!( + v, + json!({ + "kind": "userPrompt", + "text": "describe this", + "attachments": ["clipboard.png"] + }) + ); +} + #[test] fn reply_chunk_text_delta_serialises_exact_camel_case() { let v = serde_json::to_value(ReplyChunk::TextDelta { @@ -105,7 +123,10 @@ fn reply_chunk_error_serialises_exact_camel_case() { #[test] fn reply_chunk_round_trips_through_json_for_every_variant() { for chunk in [ - ReplyChunk::UserPrompt { text: "u".into() }, + ReplyChunk::UserPrompt { + text: "u".into(), + attachments: vec!["clipboard.png".into()], + }, ReplyChunk::TextDelta { text: "x".into() }, ReplyChunk::ToolActivity { label: "runs".into(), @@ -140,7 +161,13 @@ fn reply_chunk_deserialises_from_camel_case_wire_payload() { // The shape the frontend (or a mock gateway) emits. let back: ReplyChunk = serde_json::from_value(json!({ "kind": "userPrompt", "text": "hi" })).unwrap(); - assert_eq!(back, ReplyChunk::UserPrompt { text: "hi".into() }); + assert_eq!( + back, + ReplyChunk::UserPrompt { + text: "hi".into(), + attachments: Vec::new() + } + ); } #[test] @@ -310,7 +337,10 @@ fn reattach_chat_dto_serialises_camel_case_with_typed_scrollback() { let dto = ReattachChatDto { session_id: "sess-1".into(), scrollback: vec![ - ReplyChunk::UserPrompt { text: "Hi".into() }, + ReplyChunk::UserPrompt { + text: "Hi".into(), + attachments: Vec::new(), + }, ReplyChunk::TextDelta { text: "Hi".into() }, ReplyChunk::Final { content: "Hi".into(), diff --git a/crates/application/tests/chat_attachments.rs b/crates/application/tests/chat_attachments.rs index 54010961..73fdb17c 100644 --- a/crates/application/tests/chat_attachments.rs +++ b/crates/application/tests/chat_attachments.rs @@ -318,7 +318,9 @@ async fn import_chat_attachments_rejects_empty_clipboard_bytes_before_store_call .unwrap_err(); assert_eq!(err.code(), "INVALID"); - assert!(err.to_string().contains("chat attachment must not be empty")); + assert!(err + .to_string() + .contains("chat attachment must not be empty")); assert!(store.path_calls.lock().unwrap().is_empty()); assert!(store.bytes_calls.lock().unwrap().is_empty()); } diff --git a/crates/backend/src/dto.rs b/crates/backend/src/dto.rs index 9a03a754..f4b3476c 100644 --- a/crates/backend/src/dto.rs +++ b/crates/backend/src/dto.rs @@ -3347,6 +3347,9 @@ pub enum ReplyChunk { UserPrompt { /// Submitted prompt text. text: String, + /// Display labels for attachments submitted with the prompt. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + attachments: Vec, }, /// An assistant text fragment (incremental chat rendering). #[serde(rename_all = "camelCase")] diff --git a/frontend/src/domain/index.ts b/frontend/src/domain/index.ts index a676a890..7c5ab8b3 100644 --- a/frontend/src/domain/index.ts +++ b/frontend/src/domain/index.ts @@ -1674,7 +1674,7 @@ export interface ReplyProgress { } export type ReplyChunk = - | { kind: "userPrompt"; text: string } + | { kind: "userPrompt"; text: string; attachments?: string[] } | { kind: "textDelta"; text: string } | { kind: "toolActivity"; label: string } | { kind: "progress"; progress: ReplyProgress } diff --git a/frontend/src/features/agents/CustomAgentChatView.test.tsx b/frontend/src/features/agents/CustomAgentChatView.test.tsx index 5f7b7325..a3a200c0 100644 --- a/frontend/src/features/agents/CustomAgentChatView.test.tsx +++ b/frontend/src/features/agents/CustomAgentChatView.test.tsx @@ -57,6 +57,50 @@ function addPluginCommand( } describe("CustomAgentChatView", () => { + it("repaints attachment labels from user prompt scrollback", async () => { + const agent = { + launchAgentChat: vi.fn(), + reattachAgentChat: vi.fn(async (sessionId: string) => ({ + sessionId, + scrollback: [ + { + kind: "userPrompt" as const, + text: "describe this", + attachments: ["clipboard.png"], + }, + ], + })), + sendAgentChat: vi.fn(async () => {}), + cancelAgentChat: vi.fn(async () => {}), + closeAgentChat: vi.fn(async () => {}), + }; + + render( + null) }, + } as unknown as Gateways} + > + + , + ); + + expect(await screen.findByText("describe this")).toBeTruthy(); + expect(screen.getByText("Fichier: clipboard.png")).toBeTruthy(); + }); + it("cancels only the current turn and keeps the structured session alive", async () => { const agent = { launchAgentChat: vi.fn(), @@ -1406,6 +1450,181 @@ describe("CustomAgentChatView", () => { ).toBe(true); }); + it("falls back to the async clipboard API when pasted images are absent from DataTransfer", async () => { + const originalClipboard = Object.getOwnPropertyDescriptor( + window.navigator, + "clipboard", + ); + const read = vi.fn(async () => [ + { + types: ["image/png"], + getType: vi.fn(async () => new Blob([new Uint8Array([1, 2, 3])], { + type: "image/png", + })), + }, + ]); + Object.defineProperty(window.navigator, "clipboard", { + configurable: true, + value: { read }, + }); + const agent = { + launchAgentChat: vi.fn(), + reattachAgentChat: vi.fn(async (sessionId: string) => ({ + sessionId, + scrollback: [], + })), + sendAgentChat: vi.fn(async () => {}), + cancelAgentChat: vi.fn(async () => {}), + closeAgentChat: vi.fn(async () => {}), + }; + + try { + render( + null) }, + } as unknown as Gateways} + > + + , + ); + + await waitFor(() => + expect(agent.reattachAgentChat).toHaveBeenCalledWith( + "chat-session-1", + expect.any(Function), + ), + ); + + fireEvent.paste(screen.getByLabelText(/message CLI custom/), { + clipboardData: { + items: [], + files: [], + getData: () => "", + }, + }); + + await screen.findByText(/Fichier joint: clipboard-image-\d+\.png/); + expect(read).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole("button", { name: "Envoyer" })); + + await waitFor(() => expect(agent.sendAgentChat).toHaveBeenCalledTimes(1)); + expect(agent.sendAgentChat).toHaveBeenCalledWith( + "chat-session-1", + "", + expect.any(Function), + { + attachments: [ + { + filename: expect.stringMatching(/^clipboard-image-\d+\.png$/), + contentBase64: "AQID", + mime: "image/png", + sourceKind: "clipboard", + }, + ], + }, + ); + } finally { + if (originalClipboard) { + Object.defineProperty(window.navigator, "clipboard", originalClipboard); + } else { + Object.defineProperty(window.navigator, "clipboard", { + configurable: true, + value: undefined, + }); + } + } + }); + + it("shows a non-blocking error when the async clipboard image read fails", async () => { + const originalClipboard = Object.getOwnPropertyDescriptor( + window.navigator, + "clipboard", + ); + Object.defineProperty(window.navigator, "clipboard", { + configurable: true, + value: { read: vi.fn(async () => { throw new Error("clipboard denied"); }) }, + }); + const agent = { + launchAgentChat: vi.fn(), + reattachAgentChat: vi.fn(async (sessionId: string) => ({ + sessionId, + scrollback: [], + })), + sendAgentChat: vi.fn(async () => {}), + cancelAgentChat: vi.fn(async () => {}), + closeAgentChat: vi.fn(async () => {}), + }; + + try { + render( + null) }, + } as unknown as Gateways} + > + + , + ); + + await waitFor(() => + expect(agent.reattachAgentChat).toHaveBeenCalledWith( + "chat-session-1", + expect.any(Function), + ), + ); + + fireEvent.paste(screen.getByLabelText(/message CLI custom/), { + clipboardData: { + items: [], + files: [], + getData: () => "", + }, + }); + + expect((await screen.findByRole("alert")).textContent).toContain( + "clipboard denied", + ); + expect( + (screen.getByRole("button", { name: "Envoyer" }) as HTMLButtonElement) + .disabled, + ).toBe(true); + } finally { + if (originalClipboard) { + Object.defineProperty(window.navigator, "clipboard", originalClipboard); + } else { + Object.defineProperty(window.navigator, "clipboard", { + configurable: true, + value: undefined, + }); + } + } + }); + it("sends a pasted clipboard image without requiring text", async () => { const agent = { launchAgentChat: vi.fn(), diff --git a/frontend/src/features/agents/CustomAgentChatView.tsx b/frontend/src/features/agents/CustomAgentChatView.tsx index 3d4fa7d7..8b5ac994 100644 --- a/frontend/src/features/agents/CustomAgentChatView.tsx +++ b/frontend/src/features/agents/CustomAgentChatView.tsx @@ -134,10 +134,33 @@ function appendAgentDelta(turns: ChatTurn[], text: string): ChatTurn[] { return next; } -function appendUserPrompt(turns: ChatTurn[], text: string): ChatTurn[] { +function attachmentLabels(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string" && item.trim() !== "") + : []; +} + +function appendUserPrompt( + turns: ChatTurn[], + text: string, + attachments: string[] = [], +): ChatTurn[] { const last = turns[turns.length - 1]; - if (last?.role === "user" && last.text === text) return turns; - return [...turns, { role: "user", text }]; + if ( + last?.role === "user" && + last.text === text && + JSON.stringify(last.attachments ?? []) === JSON.stringify(attachments) + ) { + return turns; + } + return [ + ...turns, + { + role: "user", + text, + ...(attachments.length > 0 ? { attachments } : {}), + }, + ]; } function isReplyProgressSource(value: unknown): value is ReplyProgressSource { @@ -341,19 +364,21 @@ function bytesToBase64(bytes: Uint8Array): string { async function clipboardFileToAttachment(file: File): Promise { const mime = file.type || "application/octet-stream"; + const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer())); const filename = file.name || (mime.startsWith("image/") ? `clipboard-image-${Date.now()}.${fileExtension(mime)}` : `clipboard-file-${Date.now()}`); - const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer())); return { id: `clipboard-${Date.now()}-${Math.random().toString(36).slice(2)}`, label: filename, previewUrl: mime.startsWith("image/") ? `data:${mime};base64,${contentBase64}` : undefined, - dedupeKey: fileMetadataKey(file), + dedupeKey: file.name + ? fileMetadataKey(file) + : `${mime}\u0000${file.size}\u0000${contentBase64}`, input: { filename, contentBase64, @@ -367,12 +392,12 @@ function fileMetadataKey(file: File): string { return `${file.name}\u0000${file.type}\u0000${file.size}\u0000${file.lastModified}`; } -function clipboardFiles(event: ClipboardEvent): File[] { - const clipboardItems = event.clipboardData.items - ? Array.from(event.clipboardData.items) +function dataTransferFiles(clipboardData: DataTransfer): File[] { + const clipboardItems = clipboardData.items + ? Array.from(clipboardData.items) : []; - const clipboardFiles = event.clipboardData.files - ? Array.from(event.clipboardData.files) + const clipboardFiles = clipboardData.files + ? Array.from(clipboardData.files) : []; const fromItems = clipboardItems .filter((item) => item.kind === "file") @@ -387,6 +412,25 @@ function clipboardFiles(event: ClipboardEvent): File[] { }); } +async function asyncClipboardImageFiles(): Promise { + if ( + typeof navigator === "undefined" || + typeof navigator.clipboard?.read !== "function" + ) { + return []; + } + const items = await navigator.clipboard.read(); + const files: File[] = []; + for (const item of items) { + for (const type of item.types) { + if (!type.startsWith("image/")) continue; + const blob = await item.getType(type); + files.push(new File([blob], "", { type, lastModified: 0 })); + } + } + return files; +} + function slashCommandPrefix(draft: string): string | null { if (!draft.startsWith("/")) return null; if (/\s/.test(draft)) return null; @@ -453,7 +497,11 @@ function foldChunk(turns: ChatTurn[], raw: unknown): ChatTurn[] { } case "userPrompt": case "UserPrompt": - return appendUserPrompt(turns, String(raw.text ?? raw.prompt ?? "")); + return appendUserPrompt( + turns, + String(raw.text ?? raw.prompt ?? ""), + attachmentLabels(raw.attachments), + ); default: return [...turns, { role: "unknown", text: unknownChunkLabel(raw) }]; } @@ -1059,14 +1107,17 @@ export function CustomAgentChatView({ async function pasteClipboardFiles( event: ClipboardEvent, ) { - const files = clipboardFiles(event); - if (files.length === 0) return; - - event.preventDefault(); + const target = event.currentTarget; const pastedText = event.clipboardData.getData("text/plain"); - const selectionStart = event.currentTarget.selectionStart; - const selectionEnd = event.currentTarget.selectionEnd; + const selectionStart = target.selectionStart; + const selectionEnd = target.selectionEnd; + const filesFromEvent = dataTransferFiles(event.clipboardData); try { + const files = + filesFromEvent.length > 0 ? filesFromEvent : await asyncClipboardImageFiles(); + if (files.length === 0) return; + + if (filesFromEvent.length > 0) event.preventDefault(); const nextAttachments = await Promise.all( files.map(clipboardFileToAttachment), ); @@ -1084,7 +1135,7 @@ export function CustomAgentChatView({ }); return unique.length > 0 ? [...prev, ...unique] : prev; }); - if (pastedText) { + if (filesFromEvent.length > 0 && pastedText) { if (!draftHydratedRef.current) draftEditedBeforeHydrationRef.current = true; setDraft((prev) => prev.slice(0, selectionStart) +