blomios/IdeA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(chat): restaure le collage des images dans le composer custom CLI (#184)

Browse Source 
Le collage d'une image (ex. capture d'écran) échouait en environnement desktop : le presse-papier image n'est souvent pas exposé comme un File synchrone dans clipboardData.files, mais uniquement via l'Async Clipboard API. On ajoute donc un repli sur navigator.clipboard.read() (MIME image/*) quand l'événement de collage ne fournit aucun fichier. Le preventDefault() n'est émis que lorsqu'on a effectivement lu des fichiers issus de l'événement, afin de ne pas bloquer le collage texte quand le repli async ne remonte rien.

- refactor clipboardFiles(event) -> dataTransferFiles(DataTransfer) réutilisable
- dedupeKey robuste pour les images sans nom (mime + taille + contenu base64)
- ReplyChunk::UserPrompt porte désormais les labels d'attachments (DTO + accounting reply_chunk_bytes, skip_serializing_if Vec::is_empty)
- tests ciblés frontend (+219) et backend (chat_bridge/dto/attachments) verts

QA : tests ciblés verts. Réserve : environnement headless, pas de validation manuelle desktop — validation AppImage à confirmer en suivi.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Blomios
2026-08-11 16:49:28 +02:00
parent d779b24916
commit d9f949c0f2
9 changed files with 339 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
frontend/src/features/agents/CustomAgentChatView.test.tsx
View File

@ -57,6 +57,50 @@ function addPluginCommand(
}
describe("CustomAgentChatView", () => {
it("repaints attachment labels from user prompt scrollback", async () => {
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [
{
kind: "userPrompt" as const,
text: "describe this",
attachments: ["clipboard.png"],
},
],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
expect(await screen.findByText("describe this")).toBeTruthy();
expect(screen.getByText("Fichier: clipboard.png")).toBeTruthy();
});
it("cancels only the current turn and keeps the structured session alive", async () => {
const agent = {
launchAgentChat: vi.fn(),
@ -1406,6 +1450,181 @@ describe("CustomAgentChatView", () => {
).toBe(true);
});
it("falls back to the async clipboard API when pasted images are absent from DataTransfer", async () => {
const originalClipboard = Object.getOwnPropertyDescriptor(
window.navigator,
"clipboard",
);
const read = vi.fn(async () => [
{
types: ["image/png"],
getType: vi.fn(async () => new Blob([new Uint8Array([1, 2, 3])], {
type: "image/png",
})),
},
]);
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: { read },
});
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
try {
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
await waitFor(() =>
expect(agent.reattachAgentChat).toHaveBeenCalledWith(
"chat-session-1",
expect.any(Function),
),
);
fireEvent.paste(screen.getByLabelText(/message CLI custom/), {
clipboardData: {
items: [],
files: [],
getData: () => "",
},
});
await screen.findByText(/Fichier joint: clipboard-image-\d+\.png/);
expect(read).toHaveBeenCalledTimes(1);
fireEvent.click(screen.getByRole("button", { name: "Envoyer" }));
await waitFor(() => expect(agent.sendAgentChat).toHaveBeenCalledTimes(1));
expect(agent.sendAgentChat).toHaveBeenCalledWith(
"chat-session-1",
"",
expect.any(Function),
{
attachments: [
{
filename: expect.stringMatching(/^clipboard-image-\d+\.png$/),
contentBase64: "AQID",
mime: "image/png",
sourceKind: "clipboard",
},
],
},
);
} finally {
if (originalClipboard) {
Object.defineProperty(window.navigator, "clipboard", originalClipboard);
} else {
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: undefined,
});
}
}
});
it("shows a non-blocking error when the async clipboard image read fails", async () => {
const originalClipboard = Object.getOwnPropertyDescriptor(
window.navigator,
"clipboard",
);
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: { read: vi.fn(async () => { throw new Error("clipboard denied"); }) },
});
const agent = {
launchAgentChat: vi.fn(),
reattachAgentChat: vi.fn(async (sessionId: string) => ({
sessionId,
scrollback: [],
})),
sendAgentChat: vi.fn(async () => {}),
cancelAgentChat: vi.fn(async () => {}),
closeAgentChat: vi.fn(async () => {}),
};
try {
render(
<DIProvider
gateways={{
agent,
system: { pickFile: vi.fn(async () => null) },
} as unknown as Gateways}
>
<CustomAgentChatView
projectId="project-1"
agentId="agent-1"
agentName="Worker"
profile={profile}
cwd="/repo"
nodeId="node-1"
sessionId="chat-session-1"
conversationId="conversation-1"
onSessionId={vi.fn()}
onConversationId={vi.fn()}
/>
</DIProvider>,
);
await waitFor(() =>
expect(agent.reattachAgentChat).toHaveBeenCalledWith(
"chat-session-1",
expect.any(Function),
),
);
fireEvent.paste(screen.getByLabelText(/message CLI custom/), {
clipboardData: {
items: [],
files: [],
getData: () => "",
},
});
expect((await screen.findByRole("alert")).textContent).toContain(
"clipboard denied",
);
expect(
(screen.getByRole("button", { name: "Envoyer" }) as HTMLButtonElement)
.disabled,
).toBe(true);
} finally {
if (originalClipboard) {
Object.defineProperty(window.navigator, "clipboard", originalClipboard);
} else {
Object.defineProperty(window.navigator, "clipboard", {
configurable: true,
value: undefined,
});
}
}
});
it("sends a pasted clipboard image without requiring text", async () => {
const agent = {
launchAgentChat: vi.fn(),

87
frontend/src/features/agents/CustomAgentChatView.tsx
View File

@ -134,10 +134,33 @@ function appendAgentDelta(turns: ChatTurn[], text: string): ChatTurn[] {
return next;
}
function appendUserPrompt(turns: ChatTurn[], text: string): ChatTurn[] {
function attachmentLabels(value: unknown): string[] {
return Array.isArray(value)
? value.filter((item): item is string => typeof item === "string" && item.trim() !== "")
: [];
}
function appendUserPrompt(
turns: ChatTurn[],
text: string,
attachments: string[] = [],
): ChatTurn[] {
const last = turns[turns.length - 1];
if (last?.role === "user" && last.text === text) return turns;
return [...turns, { role: "user", text }];
if (
last?.role === "user" &&
last.text === text &&
JSON.stringify(last.attachments ?? []) === JSON.stringify(attachments)
) {
return turns;
}
return [
...turns,
{
role: "user",
text,
...(attachments.length > 0 ? { attachments } : {}),
},
];
}
function isReplyProgressSource(value: unknown): value is ReplyProgressSource {
@ -341,19 +364,21 @@ function bytesToBase64(bytes: Uint8Array): string {
async function clipboardFileToAttachment(file: File): Promise<AttachmentDraft> {
const mime = file.type || "application/octet-stream";
const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer()));
const filename =
file.name ||
(mime.startsWith("image/")
? `clipboard-image-${Date.now()}.${fileExtension(mime)}`
: `clipboard-file-${Date.now()}`);
const contentBase64 = bytesToBase64(new Uint8Array(await file.arrayBuffer()));
return {
id: `clipboard-${Date.now()}-${Math.random().toString(36).slice(2)}`,
label: filename,
previewUrl: mime.startsWith("image/")
? `data:${mime};base64,${contentBase64}`
: undefined,
dedupeKey: fileMetadataKey(file),
dedupeKey: file.name
? fileMetadataKey(file)
: `${mime}\u0000${file.size}\u0000${contentBase64}`,
input: {
filename,
contentBase64,
@ -367,12 +392,12 @@ function fileMetadataKey(file: File): string {
return `${file.name}\u0000${file.type}\u0000${file.size}\u0000${file.lastModified}`;
}
function clipboardFiles(event: ClipboardEvent<HTMLTextAreaElement>): File[] {
const clipboardItems = event.clipboardData.items
? Array.from(event.clipboardData.items)
function dataTransferFiles(clipboardData: DataTransfer): File[] {
const clipboardItems = clipboardData.items
? Array.from(clipboardData.items)
: [];
const clipboardFiles = event.clipboardData.files
? Array.from(event.clipboardData.files)
const clipboardFiles = clipboardData.files
? Array.from(clipboardData.files)
: [];
const fromItems = clipboardItems
.filter((item) => item.kind === "file")
@ -387,6 +412,25 @@ function clipboardFiles(event: ClipboardEvent<HTMLTextAreaElement>): File[] {
});
}
async function asyncClipboardImageFiles(): Promise<File[]> {
if (
typeof navigator === "undefined" ||
typeof navigator.clipboard?.read !== "function"
) {
return [];
}
const items = await navigator.clipboard.read();
const files: File[] = [];
for (const item of items) {
for (const type of item.types) {
if (!type.startsWith("image/")) continue;
const blob = await item.getType(type);
files.push(new File([blob], "", { type, lastModified: 0 }));
}
}
return files;
}
function slashCommandPrefix(draft: string): string | null {
if (!draft.startsWith("/")) return null;
if (/\s/.test(draft)) return null;
@ -453,7 +497,11 @@ function foldChunk(turns: ChatTurn[], raw: unknown): ChatTurn[] {
}
case "userPrompt":
case "UserPrompt":
return appendUserPrompt(turns, String(raw.text ?? raw.prompt ?? ""));
return appendUserPrompt(
turns,
String(raw.text ?? raw.prompt ?? ""),
attachmentLabels(raw.attachments),
);
default:
return [...turns, { role: "unknown", text: unknownChunkLabel(raw) }];
}
@ -1059,14 +1107,17 @@ export function CustomAgentChatView({
async function pasteClipboardFiles(
event: ClipboardEvent<HTMLTextAreaElement>,
) {
const files = clipboardFiles(event);
if (files.length === 0) return;
event.preventDefault();
const target = event.currentTarget;
const pastedText = event.clipboardData.getData("text/plain");
const selectionStart = event.currentTarget.selectionStart;
const selectionEnd = event.currentTarget.selectionEnd;
const selectionStart = target.selectionStart;
const selectionEnd = target.selectionEnd;
const filesFromEvent = dataTransferFiles(event.clipboardData);
try {
const files =
filesFromEvent.length > 0 ? filesFromEvent : await asyncClipboardImageFiles();
if (files.length === 0) return;
if (filesFromEvent.length > 0) event.preventDefault();
const nextAttachments = await Promise.all(
files.map(clipboardFileToAttachment),
);
@ -1084,7 +1135,7 @@ export function CustomAgentChatView({
});
return unique.length > 0 ? [...prev, ...unique] : prev;
});
if (pastedText) {
if (filesFromEvent.length > 0 && pastedText) {
if (!draftHydratedRef.current) draftEditedBeforeHydrationRef.current = true;
setDraft((prev) =>
prev.slice(0, selectionStart) +