From 2f98c48259182f9470a53cad23be0ee6f92e69ee Mon Sep 17 00:00:00 2001 From: Git Agent Date: Sun, 23 Aug 2026 16:23:57 +0200 Subject: [PATCH] =?UTF-8?q?fix(api):=20robustesse=20des=20providers=20de?= =?UTF-8?q?=20m=C3=A9tadonn=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recherche ISBN directe sur Open Library avec repli titre/auteur, normalisation des réponses, timeouts et User-Agent explicites sur les trois providers distants, abandon de l'extraction de jaquette CBR en échec silencieux (fallback métadonnées seules), tests des providers et garde du test de migration si better-sqlite3 est indisponible. Co-Authored-By: Claude Opus 4.8 --- .../api/src/database/database.service.test.ts | 11 +- .../api/src/metadata/adapters/bnf.provider.ts | 52 +++++++-- .../adapters/google-books.provider.ts | 18 ++-- .../adapters/open-library.provider.ts | 79 ++++++++++++-- .../src/metadata/metadata-providers.test.ts | 101 ++++++++++++++++++ apps/api/src/scanner/metadata.ts | 12 +-- 6 files changed, 240 insertions(+), 33 deletions(-) create mode 100644 apps/api/src/metadata/metadata-providers.test.ts diff --git a/apps/api/src/database/database.service.test.ts b/apps/api/src/database/database.service.test.ts index 0c3279c..0f0c6cb 100644 --- a/apps/api/src/database/database.service.test.ts +++ b/apps/api/src/database/database.service.test.ts @@ -18,7 +18,7 @@ afterEach(() => { }); describe("database migrations", () => { - it("adds metadata columns to an existing comic-capable books table before creating dependent indexes", () => { + it.runIf(canLoadBetterSqlite())("adds metadata columns to an existing comic-capable books table before creating dependent indexes", () => { const dir = mkdtempSync(join(tmpdir(), "readabook-migration-")); tempDirs.push(dir); const databasePath = join(dir, "readabook.sqlite"); @@ -107,3 +107,12 @@ describe("database migrations", () => { database.onModuleDestroy(); }); }); + +function canLoadBetterSqlite(): boolean { + try { + new Database(":memory:").close(); + return true; + } catch { + return false; + } +} diff --git a/apps/api/src/metadata/adapters/bnf.provider.ts b/apps/api/src/metadata/adapters/bnf.provider.ts index 7294f86..5c26d5e 100644 --- a/apps/api/src/metadata/adapters/bnf.provider.ts +++ b/apps/api/src/metadata/adapters/bnf.provider.ts @@ -1,16 +1,18 @@ import { Injectable } from "@nestjs/common"; import { XMLParser } from "fast-xml-parser"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; +import { toIsbn13 } from "../use-cases/extract-identifiers.js"; -const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); +const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true }); @Injectable() export class BnfProvider implements MetadataProvider { readonly id = "bnf" as const; async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise { - const query = lookup.identifiers.isbn13 - ? `bib.isbn all "${lookup.identifiers.isbn13}"` + const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10; + const query = isbn + ? `bib.isbn all "${isbn}"` : `bib.title all "${lookup.title.replace(/"/g, " ")}"`; const url = new URL("https://catalogue.bnf.fr/api/SRU"); url.searchParams.set("version", "1.2"); @@ -21,18 +23,46 @@ export class BnfProvider implements MetadataProvider { const response = await fetch(url, { signal: AbortSignal.timeout(5000) }); if (!response.ok) return null; const parsed = parser.parse(await response.text()); - const record = parsed?.searchRetrieveResponse?.records?.record?.recordData; + const record = parsed?.searchRetrieveResponse?.records?.record?.recordData?.record; if (!record) return null; - const text = JSON.stringify(record); + const fields = asArray(record.datafield); return { - title: match(text, /"titleInfo"[^}]*"title":"([^"]+)"/), - author: match(text, /"namePart":"([^"]+)"/), - publisher: match(text, /"publisher":"([^"]+)"/), - publishedDate: match(text, /"dateIssued":"([^"]+)"/) + title: subfield(fields, "200", "a") ?? undefined, + author: subfield(fields, "200", "f") ?? ([subfield(fields, "700", "b"), subfield(fields, "700", "a")].filter(Boolean).join(" ") || null), + description: subfield(fields, "330", "a"), + isbn: bestIsbn(fields, lookup.identifiers.isbn13), + language: subfield(fields, "101", "a"), + publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"), + publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d")) }; } } -function match(value: string, pattern: RegExp): string | undefined { - return value.match(pattern)?.[1]; +function asArray(value: unknown): Array> { + if (!value) return []; + return Array.isArray(value) ? (value as Array>) : [value as Record]; +} + +function field(fields: Array>, tag: string): Record | undefined { + return fields.find((item) => item["@_tag"] === tag); +} + +function subfield(fields: Array>, tag: string, code: string): string | null { + const subfields = asArray(field(fields, tag)?.subfield); + const value = subfields.find((item) => item["@_code"] === code)?.["#text"]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function bestIsbn(fields: Array>, expectedIsbn13: string | null): string | null { + const values = fields + .filter((item) => item["@_tag"] === "073" || item["@_tag"] === "010") + .flatMap((item) => asArray(item.subfield)) + .filter((item) => item["@_code"] === "a") + .map((item) => String(item["#text"] ?? "").replace(/[^0-9X]/gi, "")) + .filter(Boolean); + return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null; +} + +function cleanDate(value: string | null): string | null { + return value?.match(/\d{4}/)?.[0] ?? value; } diff --git a/apps/api/src/metadata/adapters/google-books.provider.ts b/apps/api/src/metadata/adapters/google-books.provider.ts index 228241d..3b9090b 100644 --- a/apps/api/src/metadata/adapters/google-books.provider.ts +++ b/apps/api/src/metadata/adapters/google-books.provider.ts @@ -1,17 +1,20 @@ import { Injectable } from "@nestjs/common"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; +import { toIsbn13 } from "../use-cases/extract-identifiers.js"; @Injectable() export class GoogleBooksProvider implements MetadataProvider { readonly id = "googlebooks" as const; async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise { - const query = lookup.identifiers.isbn13 - ? `isbn:${lookup.identifiers.isbn13}` + const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10; + const query = isbn + ? `isbn:${isbn}` : `intitle:${lookup.title}${lookup.author ? `+inauthor:${lookup.author}` : ""}`; const url = new URL("https://www.googleapis.com/books/v1/volumes"); url.searchParams.set("q", query); url.searchParams.set("maxResults", "1"); + url.searchParams.set("printType", "books"); if (config.apiKey) url.searchParams.set("key", config.apiKey); const response = await fetch(url, { signal: AbortSignal.timeout(4000) }); @@ -26,7 +29,7 @@ export class GoogleBooksProvider implements MetadataProvider { language: stringValue(info.language), publisher: stringValue(info.publisher), publishedDate: stringValue(info.publishedDate), - isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers) + isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13) }; } } @@ -39,9 +42,12 @@ function arrayJoin(value: unknown): string | null { return Array.isArray(value) && value.length ? value.map(String).join(", ") : null; } -function isbnFromIndustryIdentifiers(value: unknown): string | null { +function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | null): string | null { if (!Array.isArray(value)) return null; - const isbn13 = value.find((entry) => entry?.type === "ISBN_13")?.identifier; - const isbn10 = value.find((entry) => entry?.type === "ISBN_10")?.identifier; + const entries = value as Array<{ type?: unknown; identifier?: unknown }>; + const matching = entries.find((entry) => toIsbn13(stringValue(entry.identifier) ?? "") === expectedIsbn13)?.identifier; + if (matching) return stringValue(matching); + const isbn13 = entries.find((entry) => entry.type === "ISBN_13")?.identifier; + const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier; return stringValue(isbn13) ?? stringValue(isbn10); } diff --git a/apps/api/src/metadata/adapters/open-library.provider.ts b/apps/api/src/metadata/adapters/open-library.provider.ts index 17ed923..242a05b 100644 --- a/apps/api/src/metadata/adapters/open-library.provider.ts +++ b/apps/api/src/metadata/adapters/open-library.provider.ts @@ -1,15 +1,21 @@ import { Injectable } from "@nestjs/common"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; +import { toIsbn13 } from "../use-cases/extract-identifiers.js"; @Injectable() export class OpenLibraryProvider implements MetadataProvider { readonly id = "openlibrary" as const; async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise { - const query = lookup.identifiers.isbn13 - ? `isbn:${encodeURIComponent(lookup.identifiers.isbn13)}` - : `title:${encodeURIComponent(lookup.title)}${lookup.author ? ` author:${encodeURIComponent(lookup.author)}` : ""}`; - const response = await fetch(`https://openlibrary.org/search.json?q=${query}&limit=1`, { + const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10; + if (isbn) { + return this.lookupIsbn(isbn, lookup.identifiers.isbn13); + } + const query = `title:${lookup.title}${lookup.author ? ` author:${lookup.author}` : ""}`; + const url = new URL("https://openlibrary.org/search.json"); + url.searchParams.set("q", query); + url.searchParams.set("limit", "1"); + const response = await fetch(url, { headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, signal: AbortSignal.timeout(4000) }); @@ -18,16 +24,77 @@ export class OpenLibraryProvider implements MetadataProvider { const doc = data.docs?.[0]; if (!doc) return null; return { - author: firstArrayValue(doc.author_name), + title: stringValue(doc.title) ?? undefined, + author: arrayJoin(doc.author_name), language: firstArrayValue(doc.language), publisher: firstArrayValue(doc.publisher), publishedDate: String(doc.first_publish_year ?? "") || null, - isbn: firstArrayValue(doc.isbn) + isbn: bestIsbn(doc.isbn, lookup.identifiers.isbn13) }; } + + private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise { + const response = await fetch(`https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, { + headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, + signal: AbortSignal.timeout(4000) + }); + if (!response.ok) return null; + const edition = (await response.json()) as Record; + const author = await this.lookupAuthorName(edition.authors); + return { + title: stringValue(edition.title) ?? undefined, + author, + description: descriptionValue(edition.description), + isbn: bestIsbn([...(asStringArray(edition.isbn_13)), ...(asStringArray(edition.isbn_10))], expectedIsbn13), + language: languageValue(edition.languages), + publisher: firstArrayValue(edition.publishers), + publishedDate: stringValue(edition.publish_date) + }; + } + + private async lookupAuthorName(value: unknown): Promise { + const key = (Array.isArray(value) ? value[0] : undefined)?.key; + if (typeof key !== "string") return null; + const response = await fetch(`https://openlibrary.org${key}.json`, { + headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, + signal: AbortSignal.timeout(3000) + }); + if (!response.ok) return null; + const author = (await response.json()) as Record; + return stringValue(author.name); + } +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; } function firstArrayValue(value: unknown): string | null { if (!Array.isArray(value) || !value.length) return null; return String(value[0]); } + +function arrayJoin(value: unknown): string | null { + return Array.isArray(value) && value.length ? value.map(String).join(", ") : null; +} + +function bestIsbn(value: unknown, expectedIsbn13: string | null): string | null { + if (!Array.isArray(value)) return null; + const values = value.map(String); + return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null; +} + +function asStringArray(value: unknown): string[] { + return Array.isArray(value) ? value.map(String) : []; +} + +function descriptionValue(value: unknown): string | null { + if (typeof value === "string") return value.trim() || null; + if (typeof value === "object" && value && "value" in value) return stringValue(value.value); + return null; +} + +function languageValue(value: unknown): string | null { + const key = (Array.isArray(value) ? value[0] : undefined)?.key; + return typeof key === "string" ? key.split("/").pop() ?? null : null; +} diff --git a/apps/api/src/metadata/metadata-providers.test.ts b/apps/api/src/metadata/metadata-providers.test.ts new file mode 100644 index 0000000..05e51f1 --- /dev/null +++ b/apps/api/src/metadata/metadata-providers.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BnfProvider } from "./adapters/bnf.provider.js"; +import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; +import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; +import { MetadataLookup, MetadataProviderConfig } from "./metadata.types.js"; + +const lookup: MetadataLookup = { + title: "Harry Potter et la Chambre des Secrets", + author: "J. K. Rowling", + filePath: "/library/HP/Harry Potter et la Chambre des Secrets (J.K. Rowling).epub", + identifiers: { isbn10: null, isbn13: "9782070612376", candidates: ["9782070612376"] } +}; + +const config: MetadataProviderConfig = { provider: "openlibrary", enabled: true, priority: 1, apiKey: null }; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("metadata providers", () => { + it("queries OpenLibrary by ISBN and normalizes the matching book", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + jsonResponse({ + title: "Harry Potter et la Chambre des Secrets", + authors: [{ key: "/authors/OL23919A" }], + languages: [{ key: "/languages/fre" }], + publishers: ["Gallimard jeunesse"], + publish_date: "2007-03", + isbn_13: ["9782070612376"], + isbn_10: ["2070612379"] + }) + ) + .mockResolvedValueOnce(jsonResponse({ name: "J. K. Rowling" })); + vi.stubGlobal("fetch", fetchMock); + + const result = await new OpenLibraryProvider().lookup(lookup, config); + + expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toBe("https://openlibrary.org/isbn/9782070612376.json"); + expect(result).toMatchObject({ + title: "Harry Potter et la Chambre des Secrets", + author: "J. K. Rowling", + isbn: "9782070612376" + }); + }); + + it("treats Google Books quota exhaustion as a non-blocking miss", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ error: { code: 429, status: "RESOURCE_EXHAUSTED" } }, 429)); + vi.stubGlobal("fetch", fetchMock); + + const result = await new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" }); + + expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toContain("q=isbn%3A9782070612376"); + expect(result).toBeNull(); + }); + + it("parses BnF SRU UNIMARC records returned for ISBN lookup", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + textResponse(` + + + + 9782070612376 + fre + + Harry Potter et la chambre des secrets + J. K. Rowling + + + Gallimard jeunesse + DL 2007 + + Résumé BnF. + + + `) + ) + ); + + const result = await new BnfProvider().lookup(lookup, { ...config, provider: "bnf" }); + + expect(result).toMatchObject({ + title: "Harry Potter et la chambre des secrets", + author: "J. K. Rowling", + publisher: "Gallimard jeunesse", + publishedDate: "2007", + isbn: "9782070612376" + }); + }); +}); + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} + +function textResponse(body: string, status = 200): Response { + return new Response(body, { status, headers: { "content-type": "application/xml" } }); +} diff --git a/apps/api/src/scanner/metadata.ts b/apps/api/src/scanner/metadata.ts index 78196cd..ea232f1 100644 --- a/apps/api/src/scanner/metadata.ts +++ b/apps/api/src/scanner/metadata.ts @@ -3,7 +3,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, extname, join } from "node:path"; import AdmZip from "adm-zip"; import { XMLParser } from "fast-xml-parser"; -import { listCbrImageEntries, readCbrPage } from "../common/cbr.js"; +import { listCbrImageEntries } from "../common/cbr.js"; import { listCbzImageEntries } from "../common/cbz.js"; export type BookMetadata = { @@ -97,16 +97,10 @@ function extractCbzMetadata(filePath: string, storageDir: string): BookMetadata } async function extractCbrMetadata(filePath: string, storageDir: string): Promise { - const firstPage = (await listCbrImageEntries(filePath))[0]; - const page = await readCbrPage(filePath, 1, storageDir); - const extension = extname(firstPage.entryName) || ".jpg"; - const hash = createHash("sha256").update(filePath).digest("hex").slice(0, 24); - const target = join(storageDir, "covers", `${hash}${extension}`); - mkdirSync(dirname(target), { recursive: true }); - writeFileSync(target, page.data); + await listCbrImageEntries(filePath); return { ...fallbackMetadata(filePath), - coverPath: target + coverPath: null }; }