blomios/IdeA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge(chat): restaure le collage d'image dans le composer custom CLI (#184, QA verte)

Browse Source 
Repli sur l'Async Clipboard API (navigator.clipboard.read) quand clipboardData.files est vide — cas des captures d'écran desktop. Réserve QA : validation manuelle desktop/AppImage en suivi (env headless).
This commit is contained in:
Blomios
2026-08-11 16:49:39 +02:00
parent d779b24916 d9f949c0f2
commit 2413aa7473
9 changed files with 339 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/app-tauri/src/chat.rs
View File

@ -145,7 +145,9 @@ impl ChatBridge {
fn reply_chunk_bytes(chunk: &ReplyChunk) -> usize {
match chunk {
ReplyChunk::UserPrompt { text } => text.len(),
ReplyChunk::UserPrompt { text, attachments } => {
text.len() + attachments.iter().map(String::len).sum::<usize>()
}
ReplyChunk::TextDelta { text } => text.len(),
ReplyChunk::ToolActivity { label } => label.len(),
ReplyChunk::Progress { progress } => {

5
crates/app-tauri/src/commands.rs
View File

@ -2356,6 +2356,10 @@ pub async fn agent_send(
)
.await?;
let prompt_for_model = prompt_with_imported_attachments(&prompt, &imported_attachments);
let attachment_labels = imported_attachments
.iter()
.map(|attachment| attachment.filename.clone())
.collect();
// Retain the human submit in the same live scrollback as model chunks so
// `reattach_agent_chat` can repaint the whole in-flight conversation.
@ -2363,6 +2367,7 @@ pub async fn agent_send(
&sid,
ReplyChunk::UserPrompt {
text: prompt.clone(),
attachments: attachment_labels,
},
);

4
crates/app-tauri/tests/chat_bridge.rs
View File

@ -61,7 +61,9 @@ fn error_chunk(s: &str) -> ReplyChunk {
fn chunk_bytes(chunk: &ReplyChunk) -> usize {
match chunk {
ReplyChunk::UserPrompt { text } => text.len(),
ReplyChunk::UserPrompt { text, attachments } => {
text.len() + attachments.iter().map(String::len).sum::<usize>()
}
ReplyChunk::TextDelta { text } => text.len(),
ReplyChunk::ToolActivity { label } => label.len(),
ReplyChunk::Progress { progress } => {

36
crates/app-tauri/tests/dto_chat.rs
View File

@ -27,11 +27,29 @@ use uuid::Uuid;
fn reply_chunk_user_prompt_serialises_exact_camel_case() {
let v = serde_json::to_value(ReplyChunk::UserPrompt {
text: "run tests".into(),
attachments: Vec::new(),
})
.unwrap();
assert_eq!(v, json!({ "kind": "userPrompt", "text": "run tests" }));
}
#[test]
fn reply_chunk_user_prompt_serialises_attachment_labels() {
let v = serde_json::to_value(ReplyChunk::UserPrompt {
text: "describe this".into(),
attachments: vec!["clipboard.png".into()],
})
.unwrap();
assert_eq!(
v,
json!({
"kind": "userPrompt",
"text": "describe this",
"attachments": ["clipboard.png"]
})
);
}
#[test]
fn reply_chunk_text_delta_serialises_exact_camel_case() {
let v = serde_json::to_value(ReplyChunk::TextDelta {
@ -105,7 +123,10 @@ fn reply_chunk_error_serialises_exact_camel_case() {
#[test]
fn reply_chunk_round_trips_through_json_for_every_variant() {
for chunk in [
ReplyChunk::UserPrompt { text: "u".into() },
ReplyChunk::UserPrompt {
text: "u".into(),
attachments: vec!["clipboard.png".into()],
},
ReplyChunk::TextDelta { text: "x".into() },
ReplyChunk::ToolActivity {
label: "runs".into(),
@ -140,7 +161,13 @@ fn reply_chunk_deserialises_from_camel_case_wire_payload() {
// The shape the frontend (or a mock gateway) emits.
let back: ReplyChunk =
serde_json::from_value(json!({ "kind": "userPrompt", "text": "hi" })).unwrap();
assert_eq!(back, ReplyChunk::UserPrompt { text: "hi".into() });
assert_eq!(
back,
ReplyChunk::UserPrompt {
text: "hi".into(),
attachments: Vec::new()
}
);
}
#[test]
@ -310,7 +337,10 @@ fn reattach_chat_dto_serialises_camel_case_with_typed_scrollback() {
let dto = ReattachChatDto {
session_id: "sess-1".into(),
scrollback: vec![
ReplyChunk::UserPrompt { text: "Hi".into() },
ReplyChunk::UserPrompt {
text: "Hi".into(),
attachments: Vec::new(),
},
ReplyChunk::TextDelta { text: "Hi".into() },
ReplyChunk::Final {
content: "Hi".into(),

4
crates/application/tests/chat_attachments.rs
View File

@ -318,7 +318,9 @@ async fn import_chat_attachments_rejects_empty_clipboard_bytes_before_store_call
.unwrap_err();
assert_eq!(err.code(), "INVALID");
assert!(err.to_string().contains("chat attachment must not be empty"));
assert!(err
.to_string()
.contains("chat attachment must not be empty"));
assert!(store.path_calls.lock().unwrap().is_empty());
assert!(store.bytes_calls.lock().unwrap().is_empty());
}

3
crates/backend/src/dto.rs
View File

@ -3347,6 +3347,9 @@ pub enum ReplyChunk {
UserPrompt {
/// Submitted prompt text.
text: String,
/// Display labels for attachments submitted with the prompt.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
attachments: Vec<String>,
},
/// An assistant text fragment (incremental chat rendering).
#[serde(rename_all = "camelCase")]

2
frontend/src/domain/index.ts
View File

@ -1674,7 +1674,7 @@ export interface ReplyProgress {
}
export type ReplyChunk =
| { kind: "userPrompt"; text: string }
| { kind: "userPrompt"; text: string; attachments?: string[] }
| { kind: "textDelta"; text: string }
| { kind: "toolActivity"; label: string }
| { kind: "progress"; progress: ReplyProgress }

219
frontend/src/features/agents/CustomAgentChatView.test.tsx
View File

@ -57,6 +57,50 @@ function addPluginCommand(
}
describe("CustomAgentChatView", () => {
it("repaints attachment labels from user prompt scrollback", async () => {
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [
{
kind: "userPrompt" as const,
text: "describe this",
attachments: ["clipboard.png"],
},
],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
expect(await screen.findByText("describe this")).toBeTruthy();
expect(screen.getByText("Fichier: clipboard.png")).toBeTruthy();
});
it("cancels only the current turn and keeps the structured session alive", async () => {
const agent = {
launchAgentChat: vi.fn(),
@ -1406,6 +1450,181 @@ describe("CustomAgentChatView", () => {
).toBe(true);
});
it("falls back to the async clipboard API when pasted images are absent from DataTransfer", async () => {
const originalClipboard = Object.getOwnPropertyDescriptor(
window.navigator,
"clipboard",
);
const read = vi.fn(async () => [
{
types: ["image/png"],
getType: vi.fn(async () => new Blob([new Uint8Array([1, 2, 3])], {
type: "image/png",
})),
},
]);
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: { read },
});
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
try {
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
await waitFor(() =>
expect(agent.reattachAgentChat).toHaveBeenCalledWith(
"chat-session-1",
expect.any(Function),
),
);
fireEvent.paste(screen.getByLabelText(/message CLI custom/), {
clipboardData: {
items: [],
files: [],
getData: () => "",
},
});
await screen.findByText(/Fichier joint: clipboard-image-\d+\.png/);
expect(read).toHaveBeenCalledTimes(1);
fireEvent.click(screen.getByRole("button", { name: "Envoyer" }));
await waitFor(() => expect(agent.sendAgentChat).toHaveBeenCalledTimes(1));
expect(agent.sendAgentChat).toHaveBeenCalledWith(
"chat-session-1",
"",
expect.any(Function),
{
attachments: [
{
filename: expect.stringMatching(/^clipboard-image-\d+\.png$/),
contentBase64: "AQID",
mime: "image/png",
sourceKind: "clipboard",
},
],
},
);
} finally {
if (originalClipboard) {
Object.defineProperty(window.navigator, "clipboard", originalClipboard);
} else {
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: undefined,
});
}
}
});
it("shows a non-blocking error when the async clipboard image read fails", async () => {
const originalClipboard = Object.getOwnPropertyDescriptor(
window.navigator,
"clipboard",
);
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: { read: vi.fn(async () => { throw new Error("clipboard denied"); }) },
});
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
try {
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
await waitFor(() =>
expect(agent.reattachAgentChat).toHaveBeenCalledWith(
"chat-session-1",
expect.any(Function),
),
);
fireEvent.paste(screen.getByLabelText(/message CLI custom/), {
clipboardData: {
items: [],
files: [],
getData: () => "",
},
});
expect((await screen.findByRole("alert")).textContent).toContain(
"clipboard denied",
);
expect(
(screen.getByRole("button", { name: "Envoyer" }) as HTMLButtonElement)
.disabled,
).toBe(true);
} finally {
if (originalClipboard) {
Object.defineProperty(window.navigator, "clipboard", originalClipboard);
} else {
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: undefined,
});
}
}
});
it("sends a pasted clipboard image without requiring text", async () => {
const agent = {
launchAgentChat: vi.fn(),

87
frontend/src/features/agents/CustomAgentChatView.tsx
View File

@ -134,10 +134,33 @@ function appendAgentDelta(turns: ChatTurn[], text: string): ChatTurn[] {
return next;
}
function appendUserPrompt(turns: ChatTurn[], text: string): ChatTurn[] {
function attachmentLabels(value: unknown): string[] {
return Array.isArray(value)
? value.filter((item): item is string => typeof item === "string" && item.trim() !== "")
: [];
}
function appendUserPrompt(
turns: ChatTurn[],
text: string,
attachments: string[] = [],
): ChatTurn[] {
const last = turns[turns.length - 1];
if (last?.role === "user" && last.text === text) return turns;
return [...turns, { role: "user", text }];
if (
last?.role === "user" &&
last.text === text &&
JSON.stringify(last.attachments ?? []) === JSON.stringify(attachments)
) {
return turns;
}
return [
...turns,
{
role: "user",
text,
...(attachments.length > 0 ? { attachments } : {}),
},
];
}
function isReplyProgressSource(value: unknown): value is ReplyProgressSource {
@ -341,19 +364,21 @@ function bytesToBase64(bytes: Uint8Array): string {
async function clipboardFileToAttachment(file: File): Promise<AttachmentDraft> {
const mime = file.type || "application/octet-stream";
const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer()));
const filename =
file.name ||
(mime.startsWith("image/")
? `clipboard-image-${Date.now()}.${fileExtension(mime)}`
: `clipboard-file-${Date.now()}`);
const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer()));
return {
id: `clipboard-${Date.now()}-${Math.random().toString(36).slice(2)}`,
label: filename,
previewUrl: mime.startsWith("image/")
? `data:${mime};base64,${contentBase64}`
: undefined,
dedupeKey: fileMetadataKey(file),
dedupeKey: file.name
? fileMetadataKey(file)
: `${mime}\u0000${file.size}\u0000${contentBase64}`,
input: {
filename,
contentBase64,
@ -367,12 +392,12 @@ function fileMetadataKey(file: File): string {
return `${file.name}\u0000${file.type}\u0000${file.size}\u0000${file.lastModified}`;
}
function clipboardFiles(event: ClipboardEvent<HTMLTextAreaElement>): File[] {
const clipboardItems = event.clipboardData.items
? Array.from(event.clipboardData.items)
function dataTransferFiles(clipboardData: DataTransfer): File[] {
const clipboardItems = clipboardData.items
? Array.from(clipboardData.items)
: [];
const clipboardFiles = event.clipboardData.files
? Array.from(event.clipboardData.files)
const clipboardFiles = clipboardData.files
? Array.from(clipboardData.files)
: [];
const fromItems = clipboardItems
.filter((item) => item.kind === "file")
@ -387,6 +412,25 @@ function clipboardFiles(event: ClipboardEvent<HTMLTextAreaElement>): File[] {
});
}
async function asyncClipboardImageFiles(): Promise<File[]> {
if (
typeof navigator === "undefined" ||
typeof navigator.clipboard?.read !== "function"
) {
return [];
}
const items = await navigator.clipboard.read();
const files: File[] = [];
for (const item of items) {
for (const type of item.types) {
if (!type.startsWith("image/")) continue;
const blob = await item.getType(type);
files.push(new File([blob], "", { type, lastModified: 0 }));
}
}
return files;
}
function slashCommandPrefix(draft: string): string | null {
if (!draft.startsWith("/")) return null;
if (/\s/.test(draft)) return null;
@ -453,7 +497,11 @@ function foldChunk(turns: ChatTurn[], raw: unknown): ChatTurn[] {
}
case "userPrompt":
case "UserPrompt":
return appendUserPrompt(turns, String(raw.text ?? raw.prompt ?? ""));
return appendUserPrompt(
turns,
String(raw.text ?? raw.prompt ?? ""),
attachmentLabels(raw.attachments),
);
default:
return [...turns, { role: "unknown", text: unknownChunkLabel(raw) }];
}
@ -1059,14 +1107,17 @@ export function CustomAgentChatView({
async function pasteClipboardFiles(
event: ClipboardEvent<HTMLTextAreaElement>,
) {
const files = clipboardFiles(event);
if (files.length === 0) return;
event.preventDefault();
const target = event.currentTarget;
const pastedText = event.clipboardData.getData("text/plain");
const selectionStart = event.currentTarget.selectionStart;
const selectionEnd = event.currentTarget.selectionEnd;
const selectionStart = target.selectionStart;
const selectionEnd = target.selectionEnd;
const filesFromEvent = dataTransferFiles(event.clipboardData);
try {
const files =
filesFromEvent.length > 0 ? filesFromEvent : await asyncClipboardImageFiles();
if (files.length === 0) return;
if (filesFromEvent.length > 0) event.preventDefault();
const nextAttachments = await Promise.all(
files.map(clipboardFileToAttachment),
);
@ -1084,7 +1135,7 @@ export function CustomAgentChatView({
});
return unique.length > 0 ? [...prev, ...unique] : prev;
});
if (pastedText) {
if (filesFromEvent.length > 0 && pastedText) {
if (!draftHydratedRef.current) draftEditedBeforeHydrationRef.current = true;
setDraft((prev) =>
prev.slice(0, selectionStart) +