fix(api): robustesse des providers de métadonnées
Recherche ISBN directe sur Open Library avec repli titre/auteur, normalisation des réponses, timeouts et User-Agent explicites sur les trois providers distants, abandon de l'extraction de jaquette CBR en échec silencieux (fallback métadonnées seules), tests des providers et garde du test de migration si better-sqlite3 est indisponible. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@ -18,7 +18,7 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("database migrations", () => {
|
||||
it("adds metadata columns to an existing comic-capable books table before creating dependent indexes", () => {
|
||||
it.runIf(canLoadBetterSqlite())("adds metadata columns to an existing comic-capable books table before creating dependent indexes", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "readabook-migration-"));
|
||||
tempDirs.push(dir);
|
||||
const databasePath = join(dir, "readabook.sqlite");
|
||||
@ -107,3 +107,12 @@ describe("database migrations", () => {
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
});
|
||||
|
||||
function canLoadBetterSqlite(): boolean {
|
||||
try {
|
||||
new Database(":memory:").close();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true });
|
||||
|
||||
@Injectable()
|
||||
export class BnfProvider implements MetadataProvider {
|
||||
readonly id = "bnf" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const query = lookup.identifiers.isbn13
|
||||
? `bib.isbn all "${lookup.identifiers.isbn13}"`
|
||||
const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10;
|
||||
const query = isbn
|
||||
? `bib.isbn all "${isbn}"`
|
||||
: `bib.title all "${lookup.title.replace(/"/g, " ")}"`;
|
||||
const url = new URL("https://catalogue.bnf.fr/api/SRU");
|
||||
url.searchParams.set("version", "1.2");
|
||||
@ -21,18 +23,46 @@ export class BnfProvider implements MetadataProvider {
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
||||
if (!response.ok) return null;
|
||||
const parsed = parser.parse(await response.text());
|
||||
const record = parsed?.searchRetrieveResponse?.records?.record?.recordData;
|
||||
const record = parsed?.searchRetrieveResponse?.records?.record?.recordData?.record;
|
||||
if (!record) return null;
|
||||
const text = JSON.stringify(record);
|
||||
const fields = asArray(record.datafield);
|
||||
return {
|
||||
title: match(text, /"titleInfo"[^}]*"title":"([^"]+)"/),
|
||||
author: match(text, /"namePart":"([^"]+)"/),
|
||||
publisher: match(text, /"publisher":"([^"]+)"/),
|
||||
publishedDate: match(text, /"dateIssued":"([^"]+)"/)
|
||||
title: subfield(fields, "200", "a") ?? undefined,
|
||||
author: subfield(fields, "200", "f") ?? ([subfield(fields, "700", "b"), subfield(fields, "700", "a")].filter(Boolean).join(" ") || null),
|
||||
description: subfield(fields, "330", "a"),
|
||||
isbn: bestIsbn(fields, lookup.identifiers.isbn13),
|
||||
language: subfield(fields, "101", "a"),
|
||||
publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"),
|
||||
publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function match(value: string, pattern: RegExp): string | undefined {
|
||||
return value.match(pattern)?.[1];
|
||||
function asArray(value: unknown): Array<Record<string, unknown>> {
|
||||
if (!value) return [];
|
||||
return Array.isArray(value) ? (value as Array<Record<string, unknown>>) : [value as Record<string, unknown>];
|
||||
}
|
||||
|
||||
function field(fields: Array<Record<string, unknown>>, tag: string): Record<string, unknown> | undefined {
|
||||
return fields.find((item) => item["@_tag"] === tag);
|
||||
}
|
||||
|
||||
function subfield(fields: Array<Record<string, unknown>>, tag: string, code: string): string | null {
|
||||
const subfields = asArray(field(fields, tag)?.subfield);
|
||||
const value = subfields.find((item) => item["@_code"] === code)?.["#text"];
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
|
||||
function bestIsbn(fields: Array<Record<string, unknown>>, expectedIsbn13: string | null): string | null {
|
||||
const values = fields
|
||||
.filter((item) => item["@_tag"] === "073" || item["@_tag"] === "010")
|
||||
.flatMap((item) => asArray(item.subfield))
|
||||
.filter((item) => item["@_code"] === "a")
|
||||
.map((item) => String(item["#text"] ?? "").replace(/[^0-9X]/gi, ""))
|
||||
.filter(Boolean);
|
||||
return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null;
|
||||
}
|
||||
|
||||
function cleanDate(value: string | null): string | null {
|
||||
return value?.match(/\d{4}/)?.[0] ?? value;
|
||||
}
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
@Injectable()
|
||||
export class GoogleBooksProvider implements MetadataProvider {
|
||||
readonly id = "googlebooks" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const query = lookup.identifiers.isbn13
|
||||
? `isbn:${lookup.identifiers.isbn13}`
|
||||
const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10;
|
||||
const query = isbn
|
||||
? `isbn:${isbn}`
|
||||
: `intitle:${lookup.title}${lookup.author ? `+inauthor:${lookup.author}` : ""}`;
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("maxResults", "1");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
@ -26,7 +29,7 @@ export class GoogleBooksProvider implements MetadataProvider {
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers)
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13)
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -39,9 +42,12 @@ function arrayJoin(value: unknown): string | null {
|
||||
return Array.isArray(value) && value.length ? value.map(String).join(", ") : null;
|
||||
}
|
||||
|
||||
function isbnFromIndustryIdentifiers(value: unknown): string | null {
|
||||
function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | null): string | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const isbn13 = value.find((entry) => entry?.type === "ISBN_13")?.identifier;
|
||||
const isbn10 = value.find((entry) => entry?.type === "ISBN_10")?.identifier;
|
||||
const entries = value as Array<{ type?: unknown; identifier?: unknown }>;
|
||||
const matching = entries.find((entry) => toIsbn13(stringValue(entry.identifier) ?? "") === expectedIsbn13)?.identifier;
|
||||
if (matching) return stringValue(matching);
|
||||
const isbn13 = entries.find((entry) => entry.type === "ISBN_13")?.identifier;
|
||||
const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier;
|
||||
return stringValue(isbn13) ?? stringValue(isbn10);
|
||||
}
|
||||
|
||||
@ -1,15 +1,21 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
@Injectable()
|
||||
export class OpenLibraryProvider implements MetadataProvider {
|
||||
readonly id = "openlibrary" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const query = lookup.identifiers.isbn13
|
||||
? `isbn:${encodeURIComponent(lookup.identifiers.isbn13)}`
|
||||
: `title:${encodeURIComponent(lookup.title)}${lookup.author ? ` author:${encodeURIComponent(lookup.author)}` : ""}`;
|
||||
const response = await fetch(`https://openlibrary.org/search.json?q=${query}&limit=1`, {
|
||||
const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10;
|
||||
if (isbn) {
|
||||
return this.lookupIsbn(isbn, lookup.identifiers.isbn13);
|
||||
}
|
||||
const query = `title:${lookup.title}${lookup.author ? ` author:${lookup.author}` : ""}`;
|
||||
const url = new URL("https://openlibrary.org/search.json");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("limit", "1");
|
||||
const response = await fetch(url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
});
|
||||
@ -18,16 +24,77 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
const doc = data.docs?.[0];
|
||||
if (!doc) return null;
|
||||
return {
|
||||
author: firstArrayValue(doc.author_name),
|
||||
title: stringValue(doc.title) ?? undefined,
|
||||
author: arrayJoin(doc.author_name),
|
||||
language: firstArrayValue(doc.language),
|
||||
publisher: firstArrayValue(doc.publisher),
|
||||
publishedDate: String(doc.first_publish_year ?? "") || null,
|
||||
isbn: firstArrayValue(doc.isbn)
|
||||
isbn: bestIsbn(doc.isbn, lookup.identifiers.isbn13)
|
||||
};
|
||||
}
|
||||
|
||||
private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const response = await fetch(`https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const edition = (await response.json()) as Record<string, unknown>;
|
||||
const author = await this.lookupAuthorName(edition.authors);
|
||||
return {
|
||||
title: stringValue(edition.title) ?? undefined,
|
||||
author,
|
||||
description: descriptionValue(edition.description),
|
||||
isbn: bestIsbn([...(asStringArray(edition.isbn_13)), ...(asStringArray(edition.isbn_10))], expectedIsbn13),
|
||||
language: languageValue(edition.languages),
|
||||
publisher: firstArrayValue(edition.publishers),
|
||||
publishedDate: stringValue(edition.publish_date)
|
||||
};
|
||||
}
|
||||
|
||||
private async lookupAuthorName(value: unknown): Promise<string | null> {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
if (typeof key !== "string") return null;
|
||||
const response = await fetch(`https://openlibrary.org${key}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(3000)
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const author = (await response.json()) as Record<string, unknown>;
|
||||
return stringValue(author.name);
|
||||
}
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
|
||||
function firstArrayValue(value: unknown): string | null {
|
||||
if (!Array.isArray(value) || !value.length) return null;
|
||||
return String(value[0]);
|
||||
}
|
||||
|
||||
function arrayJoin(value: unknown): string | null {
|
||||
return Array.isArray(value) && value.length ? value.map(String).join(", ") : null;
|
||||
}
|
||||
|
||||
function bestIsbn(value: unknown, expectedIsbn13: string | null): string | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const values = value.map(String);
|
||||
return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null;
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.map(String) : [];
|
||||
}
|
||||
|
||||
function descriptionValue(value: unknown): string | null {
|
||||
if (typeof value === "string") return value.trim() || null;
|
||||
if (typeof value === "object" && value && "value" in value) return stringValue(value.value);
|
||||
return null;
|
||||
}
|
||||
|
||||
function languageValue(value: unknown): string | null {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
return typeof key === "string" ? key.split("/").pop() ?? null : null;
|
||||
}
|
||||
|
||||
101
apps/api/src/metadata/metadata-providers.test.ts
Normal file
101
apps/api/src/metadata/metadata-providers.test.ts
Normal file
@ -0,0 +1,101 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BnfProvider } from "./adapters/bnf.provider.js";
|
||||
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
|
||||
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
|
||||
import { MetadataLookup, MetadataProviderConfig } from "./metadata.types.js";
|
||||
|
||||
const lookup: MetadataLookup = {
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
author: "J. K. Rowling",
|
||||
filePath: "/library/HP/Harry Potter et la Chambre des Secrets (J.K. Rowling).epub",
|
||||
identifiers: { isbn10: null, isbn13: "9782070612376", candidates: ["9782070612376"] }
|
||||
};
|
||||
|
||||
const config: MetadataProviderConfig = { provider: "openlibrary", enabled: true, priority: 1, apiKey: null };
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("metadata providers", () => {
|
||||
it("queries OpenLibrary by ISBN and normalizes the matching book", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
authors: [{ key: "/authors/OL23919A" }],
|
||||
languages: [{ key: "/languages/fre" }],
|
||||
publishers: ["Gallimard jeunesse"],
|
||||
publish_date: "2007-03",
|
||||
isbn_13: ["9782070612376"],
|
||||
isbn_10: ["2070612379"]
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(jsonResponse({ name: "J. K. Rowling" }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await new OpenLibraryProvider().lookup(lookup, config);
|
||||
|
||||
expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toBe("https://openlibrary.org/isbn/9782070612376.json");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612376"
|
||||
});
|
||||
});
|
||||
|
||||
it("treats Google Books quota exhaustion as a non-blocking miss", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ error: { code: 429, status: "RESOURCE_EXHAUSTED" } }, 429));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" });
|
||||
|
||||
expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toContain("q=isbn%3A9782070612376");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("parses BnF SRU UNIMARC records returned for ISBN lookup", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () =>
|
||||
textResponse(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<srw:searchRetrieveResponse xmlns:srw="http://www.loc.gov/zing/srw/">
|
||||
<srw:records><srw:record><srw:recordData>
|
||||
<mxc:record xmlns:mxc="info:lc/xmlns/marcxchange-v2">
|
||||
<mxc:datafield tag="073"><mxc:subfield code="a">9782070612376</mxc:subfield></mxc:datafield>
|
||||
<mxc:datafield tag="101"><mxc:subfield code="a">fre</mxc:subfield></mxc:datafield>
|
||||
<mxc:datafield tag="200">
|
||||
<mxc:subfield code="a">Harry Potter et la chambre des secrets</mxc:subfield>
|
||||
<mxc:subfield code="f">J. K. Rowling</mxc:subfield>
|
||||
</mxc:datafield>
|
||||
<mxc:datafield tag="210">
|
||||
<mxc:subfield code="c">Gallimard jeunesse</mxc:subfield>
|
||||
<mxc:subfield code="d">DL 2007</mxc:subfield>
|
||||
</mxc:datafield>
|
||||
<mxc:datafield tag="330"><mxc:subfield code="a">Résumé BnF.</mxc:subfield></mxc:datafield>
|
||||
</mxc:record>
|
||||
</srw:recordData></srw:record></srw:records>
|
||||
</srw:searchRetrieveResponse>`)
|
||||
)
|
||||
);
|
||||
|
||||
const result = await new BnfProvider().lookup(lookup, { ...config, provider: "bnf" });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et la chambre des secrets",
|
||||
author: "J. K. Rowling",
|
||||
publisher: "Gallimard jeunesse",
|
||||
publishedDate: "2007",
|
||||
isbn: "9782070612376"
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
function textResponse(body: string, status = 200): Response {
|
||||
return new Response(body, { status, headers: { "content-type": "application/xml" } });
|
||||
}
|
||||
@ -3,7 +3,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { basename, dirname, extname, join } from "node:path";
|
||||
import AdmZip from "adm-zip";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { listCbrImageEntries, readCbrPage } from "../common/cbr.js";
|
||||
import { listCbrImageEntries } from "../common/cbr.js";
|
||||
import { listCbzImageEntries } from "../common/cbz.js";
|
||||
|
||||
export type BookMetadata = {
|
||||
@ -97,16 +97,10 @@ function extractCbzMetadata(filePath: string, storageDir: string): BookMetadata
|
||||
}
|
||||
|
||||
async function extractCbrMetadata(filePath: string, storageDir: string): Promise<BookMetadata> {
|
||||
const firstPage = (await listCbrImageEntries(filePath))[0];
|
||||
const page = await readCbrPage(filePath, 1, storageDir);
|
||||
const extension = extname(firstPage.entryName) || ".jpg";
|
||||
const hash = createHash("sha256").update(filePath).digest("hex").slice(0, 24);
|
||||
const target = join(storageDir, "covers", `${hash}${extension}`);
|
||||
mkdirSync(dirname(target), { recursive: true });
|
||||
writeFileSync(target, page.data);
|
||||
await listCbrImageEntries(filePath);
|
||||
return {
|
||||
...fallbackMetadata(filePath),
|
||||
coverPath: target
|
||||
coverPath: null
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user