fix(api,web): enrichissement métadonnées — hints locaux et scoring des correspondances
Recherche de fiches par nom peu fiable : sans ISBN, les providers étaient interrogés avec des titres bruts peu discriminants. - extraction de hints locaux (titre/auteur/année/isbn du fichier et du nom de fichier) persistés dans books.local_metadata_json - nouveau use-case ScoreMetadataMatch : tri des résultats par score de correspondance avant sélection du meilleur candidat - providers (google-books, open-library, bnf, local) durcis et nourris par les hints ; colonne + index local_metadata_json avec migrations idempotentes (création et rebuild legacy) - web : nettoyage de la description de la fiche livre (cleanBookDescription, white-space pre-line) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Git Agent
@ -58,6 +58,7 @@ export class DatabaseService implements OnModuleDestroy {
|
||||
isbn TEXT,
|
||||
isbn13 TEXT,
|
||||
identifiers_json TEXT,
|
||||
local_metadata_json TEXT,
|
||||
language TEXT,
|
||||
publisher TEXT,
|
||||
published_date TEXT,
|
||||
@ -175,6 +176,7 @@ export class DatabaseService implements OnModuleDestroy {
|
||||
isbn TEXT,
|
||||
isbn13 TEXT,
|
||||
identifiers_json TEXT,
|
||||
local_metadata_json TEXT,
|
||||
language TEXT,
|
||||
publisher TEXT,
|
||||
published_date TEXT,
|
||||
@ -187,11 +189,11 @@ export class DatabaseService implements OnModuleDestroy {
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO books (
|
||||
id, library_id, title, author, description, isbn, isbn13, identifiers_json, language, publisher, published_date,
|
||||
id, library_id, title, author, description, isbn, isbn13, identifiers_json, local_metadata_json, language, publisher, published_date,
|
||||
format, file_path, cover_path, file_size, file_mtime, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
id, library_id, title, author, description, isbn, NULL, NULL, language, publisher, published_date,
|
||||
id, library_id, title, author, description, isbn, NULL, NULL, NULL, language, publisher, published_date,
|
||||
format, file_path, cover_path, file_size, file_mtime, created_at, updated_at
|
||||
FROM books_legacy_format;
|
||||
DROP TABLE books_legacy_format;
|
||||
@ -233,7 +235,11 @@ export class DatabaseService implements OnModuleDestroy {
|
||||
if (!names.has("identifiers_json")) {
|
||||
this.sqlite.exec("ALTER TABLE books ADD COLUMN identifiers_json TEXT");
|
||||
}
|
||||
if (!names.has("local_metadata_json")) {
|
||||
this.sqlite.exec("ALTER TABLE books ADD COLUMN local_metadata_json TEXT");
|
||||
}
|
||||
this.sqlite.exec("CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13)");
|
||||
this.sqlite.exec("CREATE INDEX IF NOT EXISTS books_local_metadata_idx ON books(local_metadata_json)");
|
||||
}
|
||||
|
||||
private ensureReaderPreferencesTable(): void {
|
||||
|
||||
@ -36,6 +36,7 @@ export const books = sqliteTable(
|
||||
isbn: text("isbn"),
|
||||
isbn13: text("isbn13"),
|
||||
identifiersJson: text("identifiers_json"),
|
||||
localMetadataJson: text("local_metadata_json"),
|
||||
language: text("language"),
|
||||
publisher: text("publisher"),
|
||||
publishedDate: text("published_date"),
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true });
|
||||
@ -11,30 +11,52 @@ export class BnfProvider implements MetadataProvider {
|
||||
|
||||
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10;
|
||||
const query = isbn
|
||||
? `bib.isbn all "${isbn}"`
|
||||
: `bib.title all "${lookup.title.replace(/"/g, " ")}"`;
|
||||
if (!isbn) {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, _config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
const query = `bib.isbn all "${isbn}"`;
|
||||
const matches = await this.searchSru(query, 1, lookup.identifiers.isbn13);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, _config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const title = query.title.replace(/"/g, " ");
|
||||
const author = query.author?.replace(/"/g, " ");
|
||||
const sruQuery = [`bib.title all "${title}"`, author ? `bib.author all "${author}"` : null, query.year ? `bib.date all "${query.year}"` : null]
|
||||
.filter(Boolean)
|
||||
.join(" and ");
|
||||
return this.searchSru(sruQuery, 5, query.isbn ? toIsbn13(query.isbn) : null);
|
||||
}
|
||||
|
||||
private async searchSru(query: string, maximumRecords: number, expectedIsbn13: string | null): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://catalogue.bnf.fr/api/SRU");
|
||||
url.searchParams.set("version", "1.2");
|
||||
url.searchParams.set("operation", "searchRetrieve");
|
||||
url.searchParams.set("query", query);
|
||||
url.searchParams.set("maximumRecords", "1");
|
||||
url.searchParams.set("maximumRecords", String(maximumRecords));
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
||||
if (!response.ok) return null;
|
||||
if (!response.ok) return [];
|
||||
const parsed = parser.parse(await response.text());
|
||||
const record = parsed?.searchRetrieveResponse?.records?.record?.recordData?.record;
|
||||
if (!record) return null;
|
||||
const fields = asArray(record.datafield);
|
||||
return {
|
||||
title: subfield(fields, "200", "a") ?? undefined,
|
||||
author: subfield(fields, "200", "f") ?? ([subfield(fields, "700", "b"), subfield(fields, "700", "a")].filter(Boolean).join(" ") || null),
|
||||
description: subfield(fields, "330", "a"),
|
||||
isbn: bestIsbn(fields, lookup.identifiers.isbn13),
|
||||
language: subfield(fields, "101", "a"),
|
||||
publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"),
|
||||
publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))
|
||||
};
|
||||
const records = asArray(parsed?.searchRetrieveResponse?.records?.record)
|
||||
.map((entry) => (entry.recordData as Record<string, unknown> | undefined)?.record)
|
||||
.filter((record): record is Record<string, unknown> => Boolean(record));
|
||||
if (!records.length) return [];
|
||||
return records
|
||||
.map((record) => {
|
||||
const fields = asArray(record.datafield);
|
||||
return {
|
||||
title: subfield(fields, "200", "a") ?? undefined,
|
||||
author: subfield(fields, "200", "f") ?? ([subfield(fields, "700", "b"), subfield(fields, "700", "a")].filter(Boolean).join(" ") || null),
|
||||
description: subfield(fields, "330", "a"),
|
||||
isbn: bestIsbn(fields, expectedIsbn13),
|
||||
language: subfield(fields, "101", "a"),
|
||||
publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"),
|
||||
publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))
|
||||
};
|
||||
})
|
||||
.sort((left, right) => Number(Boolean(right.isbn)) - Number(Boolean(left.isbn)));
|
||||
}
|
||||
}
|
||||
|
||||
@ -60,7 +82,11 @@ function bestIsbn(fields: Array<Record<string, unknown>>, expectedIsbn13: string
|
||||
.filter((item) => item["@_code"] === "a")
|
||||
.map((item) => String(item["#text"] ?? "").replace(/[^0-9X]/gi, ""))
|
||||
.filter(Boolean);
|
||||
return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null;
|
||||
return (
|
||||
values.find((candidate) => Boolean(expectedIsbn13) && toIsbn13(candidate) === expectedIsbn13) ??
|
||||
values.find((candidate) => Boolean(toIsbn13(candidate))) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function cleanDate(value: string | null): string | null {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
@Injectable()
|
||||
@ -8,9 +8,11 @@ export class GoogleBooksProvider implements MetadataProvider {
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const isbn = lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10;
|
||||
const query = isbn
|
||||
? `isbn:${isbn}`
|
||||
: `intitle:${lookup.title}${lookup.author ? `+inauthor:${lookup.author}` : ""}`;
|
||||
if (!isbn) {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
const query = `isbn:${isbn}`;
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("maxResults", "1");
|
||||
@ -32,6 +34,30 @@ export class GoogleBooksProvider implements MetadataProvider {
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13)
|
||||
};
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", `intitle:${query.title}${query.author ? `+inauthor:${query.author}` : ""}`);
|
||||
url.searchParams.set("maxResults", "5");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
if (!response.ok) return [];
|
||||
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
|
||||
return (data.items ?? [])
|
||||
.map((item) => item.volumeInfo)
|
||||
.filter((info): info is Record<string, unknown> => Boolean(info))
|
||||
.map((info) => ({
|
||||
title: stringValue(info.title) ?? undefined,
|
||||
author: arrayJoin(info.authors),
|
||||
description: stringValue(info.description),
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
@ -45,7 +71,7 @@ function arrayJoin(value: unknown): string | null {
|
||||
function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | null): string | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const entries = value as Array<{ type?: unknown; identifier?: unknown }>;
|
||||
const matching = entries.find((entry) => toIsbn13(stringValue(entry.identifier) ?? "") === expectedIsbn13)?.identifier;
|
||||
const matching = entries.find((entry) => Boolean(expectedIsbn13) && toIsbn13(stringValue(entry.identifier) ?? "") === expectedIsbn13)?.identifier;
|
||||
if (matching) return stringValue(matching);
|
||||
const isbn13 = entries.find((entry) => entry.type === "ISBN_13")?.identifier;
|
||||
const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
|
||||
@Injectable()
|
||||
export class LocalMetadataProvider implements MetadataProvider {
|
||||
@ -13,4 +13,15 @@ export class LocalMetadataProvider implements MetadataProvider {
|
||||
identifiers: lookup.identifiers
|
||||
};
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, _config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
return [
|
||||
{
|
||||
title: query.title,
|
||||
author: query.author,
|
||||
isbn: query.isbn ?? null,
|
||||
publishedDate: query.year ?? null
|
||||
}
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
|
||||
@Injectable()
|
||||
@ -11,26 +11,45 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
if (isbn) {
|
||||
return this.lookupIsbn(isbn, lookup.identifiers.isbn13);
|
||||
}
|
||||
const query = `title:${lookup.title}${lookup.author ? ` author:${lookup.author}` : ""}`;
|
||||
if (lookup.sourceId) {
|
||||
return this.lookupEdition(lookup.sourceId, lookup.identifiers.isbn13);
|
||||
}
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, _config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, _config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://openlibrary.org/search.json");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("limit", "1");
|
||||
url.searchParams.set("title", query.title);
|
||||
if (query.author) url.searchParams.set("author", query.author);
|
||||
if (query.year) url.searchParams.set("first_publish_year", query.year);
|
||||
url.searchParams.set("limit", "5");
|
||||
const response = await fetch(url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (!response.ok) return [];
|
||||
const data = (await response.json()) as { docs?: Array<Record<string, unknown>> };
|
||||
const doc = data.docs?.[0];
|
||||
if (!doc) return null;
|
||||
return {
|
||||
return (data.docs ?? []).map((doc) => ({
|
||||
title: stringValue(doc.title) ?? undefined,
|
||||
sourceId: firstArrayValue(doc.edition_key) ?? stringValue(doc.cover_edition_key),
|
||||
author: arrayJoin(doc.author_name),
|
||||
language: firstArrayValue(doc.language),
|
||||
publisher: firstArrayValue(doc.publisher),
|
||||
publishedDate: String(doc.first_publish_year ?? "") || null,
|
||||
isbn: bestIsbn(doc.isbn, lookup.identifiers.isbn13)
|
||||
};
|
||||
isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
}));
|
||||
}
|
||||
|
||||
private async lookupEdition(sourceId: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const editionKey = sourceId.replace(/^\/?books\//, "");
|
||||
if (!editionKey) return null;
|
||||
const response = await fetch(`https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
@ -39,7 +58,10 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
signal: AbortSignal.timeout(4000)
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const edition = (await response.json()) as Record<string, unknown>;
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
private async editionToMatch(edition: Record<string, unknown>, expectedIsbn13: string | null): Promise<MetadataMatch> {
|
||||
const author = await this.lookupAuthorName(edition.authors);
|
||||
return {
|
||||
title: stringValue(edition.title) ?? undefined,
|
||||
@ -81,7 +103,11 @@ function arrayJoin(value: unknown): string | null {
|
||||
function bestIsbn(value: unknown, expectedIsbn13: string | null): string | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const values = value.map(String);
|
||||
return values.find((candidate) => toIsbn13(candidate) === expectedIsbn13) ?? values.find((candidate) => toIsbn13(candidate)) ?? null;
|
||||
return (
|
||||
values.find((candidate) => Boolean(expectedIsbn13) && toIsbn13(candidate) === expectedIsbn13) ??
|
||||
values.find((candidate) => Boolean(toIsbn13(candidate))) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
|
||||
@ -9,6 +9,7 @@ describe("ISBN normalization", () => {
|
||||
expect(normalizeIsbn("0-306-40615-2")).toBe("0306406152");
|
||||
expect(normalizeIsbn("978-0-306-40615-7")).toBe("9780306406157");
|
||||
expect(normalizeIsbn("978-0-306-40615-8")).toBeNull();
|
||||
expect(normalizeIsbn("5030931067112")).toBeNull();
|
||||
});
|
||||
|
||||
it("converts ISBN-10 to ISBN-13", () => {
|
||||
|
||||
43
apps/api/src/metadata/local-metadata-hints.test.ts
Normal file
43
apps/api/src/metadata/local-metadata-hints.test.ts
Normal file
@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ExtractLocalMetadataHints } from "./use-cases/extract-local-metadata-hints.js";
|
||||
import { ScoreMetadataMatch } from "./use-cases/score-metadata-match.js";
|
||||
|
||||
describe("local metadata hints", () => {
|
||||
it("extracts title, author and year hints from a book without ISBN", () => {
|
||||
const hints = new ExtractLocalMetadataHints().fromMetadataAndFile(
|
||||
{
|
||||
title: "Harry Potter et le Prince de Sang Mele",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Harry Potter et le Prince de Sang Mele (J. K. Rowling) 2005.epub"
|
||||
);
|
||||
|
||||
expect(hints).toMatchObject({
|
||||
title: "Harry Potter et le Prince de Sang Mele",
|
||||
author: "J. K. Rowling",
|
||||
year: "2005",
|
||||
isbn: null
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("metadata match scoring", () => {
|
||||
it("keeps the best remote match for locally extracted title and author", () => {
|
||||
const best = new ScoreMetadataMatch().best(
|
||||
{ title: "Harry Potter et le Prince de Sang Mele", author: "J. K. Rowling", year: "2005" },
|
||||
[
|
||||
{ title: "Harry Potter et la chambre des secrets", author: "J. K. Rowling", publishedDate: "1998" },
|
||||
{ title: "Harry Potter et le Prince de sang-mêlé", author: "J.K. Rowling", publishedDate: "2005" }
|
||||
]
|
||||
);
|
||||
|
||||
expect(best?.match.title).toBe("Harry Potter et le Prince de sang-mêlé");
|
||||
expect(best?.score).toBeGreaterThan(0.7);
|
||||
});
|
||||
});
|
||||
@ -8,7 +8,20 @@ const lookup: MetadataLookup = {
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
author: "J. K. Rowling",
|
||||
filePath: "/library/HP/Harry Potter et la Chambre des Secrets (J.K. Rowling).epub",
|
||||
identifiers: { isbn10: null, isbn13: "9782070612376", candidates: ["9782070612376"] }
|
||||
identifiers: { isbn10: null, isbn13: "9782070612376", candidates: ["9782070612376"] },
|
||||
local: {
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
author: "J. K. Rowling",
|
||||
year: null,
|
||||
isbn: "9782070612376",
|
||||
fileTitle: "Harry Potter et la Chambre des Secrets (J.K. Rowling)",
|
||||
raw: {
|
||||
title: "Harry Potter et la Chambre des Secrets",
|
||||
author: "J. K. Rowling",
|
||||
publishedDate: null,
|
||||
fileName: "Harry Potter et la Chambre des Secrets (J.K. Rowling)"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const config: MetadataProviderConfig = { provider: "openlibrary", enabled: true, priority: 1, apiKey: null };
|
||||
@ -55,6 +68,67 @@ describe("metadata providers", () => {
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("queries OpenLibrary by local metadata when ISBN is missing", async () => {
|
||||
const fetchMock = vi.fn(async () =>
|
||||
jsonResponse({
|
||||
docs: [
|
||||
{
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
author_name: ["J. K. Rowling"],
|
||||
first_publish_year: 2005,
|
||||
publisher: ["Gallimard jeunesse"],
|
||||
cover_edition_key: "OL24333986M",
|
||||
isbn: ["9782070612383"]
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await new OpenLibraryProvider().searchByMetadata(
|
||||
{ title: "Harry Potter et le Prince de Sang Mele", author: "J. K. Rowling", year: "2005" },
|
||||
config
|
||||
);
|
||||
const url = new URL(String((fetchMock.mock.calls[0] as unknown[])[0]));
|
||||
|
||||
expect(url.searchParams.get("title")).toBe("Harry Potter et le Prince de Sang Mele");
|
||||
expect(url.searchParams.get("author")).toBe("J. K. Rowling");
|
||||
expect(result[0]).toMatchObject({
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
sourceId: "OL24333986M",
|
||||
author: "J. K. Rowling",
|
||||
publishedDate: "2005"
|
||||
});
|
||||
});
|
||||
|
||||
it("looks up OpenLibrary edition details from a search result source id", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
title: "Harry Potter et le prince de sang-mele",
|
||||
authors: [{ key: "/authors/OL23919A" }],
|
||||
languages: [{ key: "/languages/fre" }],
|
||||
publishers: ["Gallimard jeunesse"],
|
||||
publish_date: "2005",
|
||||
isbn_13: ["9782070612383"],
|
||||
description: { value: "Sixième année à Poudlard." }
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(jsonResponse({ name: "J. K. Rowling" }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await new OpenLibraryProvider().lookup({ ...lookup, sourceId: "OL24333986M", identifiers: { isbn10: null, isbn13: null, candidates: [] } }, config);
|
||||
|
||||
expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toBe("https://openlibrary.org/books/OL24333986M.json");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et le prince de sang-mele",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
description: "Sixième année à Poudlard."
|
||||
});
|
||||
});
|
||||
|
||||
it("parses BnF SRU UNIMARC records returned for ISBN lookup", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
@ -90,6 +164,42 @@ describe("metadata providers", () => {
|
||||
isbn: "9782070612376"
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers BnF title search records with a valid book ISBN over non-book EAN records", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () =>
|
||||
textResponse(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<srw:searchRetrieveResponse xmlns:srw="http://www.loc.gov/zing/srw/">
|
||||
<srw:records>
|
||||
<srw:record><srw:recordData>
|
||||
<mxc:record xmlns:mxc="info:lc/xmlns/marcxchange-v2">
|
||||
<mxc:datafield tag="073"><mxc:subfield code="a">5030931067112</mxc:subfield></mxc:datafield>
|
||||
<mxc:datafield tag="200"><mxc:subfield code="a">Harry Potter et le prince de sang-mêlé</mxc:subfield><mxc:subfield code="f">Electronic arts</mxc:subfield></mxc:datafield>
|
||||
</mxc:record>
|
||||
</srw:recordData></srw:record>
|
||||
<srw:record><srw:recordData>
|
||||
<mxc:record xmlns:mxc="info:lc/xmlns/marcxchange-v2">
|
||||
<mxc:datafield tag="010"><mxc:subfield code="a">274419736X</mxc:subfield></mxc:datafield>
|
||||
<mxc:datafield tag="200"><mxc:subfield code="a">Harry Potter et le prince de sang-mêlé</mxc:subfield><mxc:subfield code="f">J. K. Rowling</mxc:subfield></mxc:datafield>
|
||||
</mxc:record>
|
||||
</srw:recordData></srw:record>
|
||||
</srw:records>
|
||||
</srw:searchRetrieveResponse>`)
|
||||
)
|
||||
);
|
||||
|
||||
const result = await new BnfProvider().searchByMetadata(
|
||||
{ title: "Harry Potter et le prince de sang mele", author: null },
|
||||
{ ...config, provider: "bnf" }
|
||||
);
|
||||
|
||||
expect(result[0]).toMatchObject({
|
||||
author: "J. K. Rowling",
|
||||
isbn: "274419736X"
|
||||
});
|
||||
expect(result[1]?.isbn).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
|
||||
233
apps/api/src/metadata/metadata.service.test.ts
Normal file
233
apps/api/src/metadata/metadata.service.test.ts
Normal file
@ -0,0 +1,233 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DatabaseService } from "../database/database.service.js";
|
||||
import { BookMetadata } from "../scanner/metadata.js";
|
||||
import { MetadataService } from "./metadata.service.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "./metadata.types.js";
|
||||
|
||||
const previousDatabasePath = process.env.DATABASE_PATH;
|
||||
const previousStorageDir = process.env.STORAGE_DIR;
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
process.env.DATABASE_PATH = previousDatabasePath;
|
||||
process.env.STORAGE_DIR = previousStorageDir;
|
||||
vi.restoreAllMocks();
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("MetadataService", () => {
|
||||
it.runIf(canLoadBetterSqlite())("backfills description from provider lookup after a metadata search hit yields an ISBN", async () => {
|
||||
const database = createDatabase();
|
||||
const localProvider = providerStub("local");
|
||||
const openLibrarySearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise<MetadataMatch[]>>(async () => [
|
||||
{
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
publishedDate: "2005"
|
||||
}
|
||||
]);
|
||||
const openLibraryLookup = vi.fn<(_: MetadataLookup, __: MetadataProviderConfig) => Promise<MetadataMatch | null>>(async (lookup) => {
|
||||
if (lookup.identifiers.isbn13 !== "9782070612383") return null;
|
||||
return {
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
publishedDate: "2005",
|
||||
description: "Harry Potter découvre l'héritage du Prince de Sang-Mêlé."
|
||||
};
|
||||
});
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
searchByMetadata: openLibrarySearch,
|
||||
lookup: openLibraryLookup
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
localProvider as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Harry Potter et le Prince de Sang Mele",
|
||||
author: "J. K. Rowling",
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
};
|
||||
|
||||
const result = await service.enrichMetadata(localMetadata, "/library/HP/Harry Potter et le Prince de Sang Mele.epub", {
|
||||
remote: true
|
||||
});
|
||||
|
||||
expect(openLibrarySearch).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup.mock.calls[0]?.[0].identifiers.isbn13).toBe("9782070612383");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
isbn: "9782070612383",
|
||||
isbn13: "9782070612383",
|
||||
description: "Harry Potter découvre l'héritage du Prince de Sang-Mêlé."
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("looks up details from a title search hit even when the hit has no ISBN", async () => {
|
||||
const database = createDatabase();
|
||||
const localProvider = providerStub("local");
|
||||
const openLibrarySearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise<MetadataMatch[]>>(async () => [
|
||||
{
|
||||
title: "Harry Potter et le prince de sang-mele",
|
||||
sourceId: "OL24333986M",
|
||||
publishedDate: "2005"
|
||||
}
|
||||
]);
|
||||
const openLibraryLookup = vi.fn<(_: MetadataLookup, __: MetadataProviderConfig) => Promise<MetadataMatch | null>>(async (lookup) => {
|
||||
if (lookup.sourceId !== "OL24333986M") return null;
|
||||
return {
|
||||
title: "Harry Potter et le prince de sang-mêlé",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
publishedDate: "2005",
|
||||
description: "Sixième année à Poudlard."
|
||||
};
|
||||
});
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
searchByMetadata: openLibrarySearch,
|
||||
lookup: openLibraryLookup
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
localProvider as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Harry Potter et le prince de sang mele",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
};
|
||||
|
||||
const result = await service.enrichMetadata(localMetadata, "/library/Harry Potter et le prince de sang mele.epub", {
|
||||
remote: true
|
||||
});
|
||||
|
||||
expect(openLibrarySearch).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup.mock.calls[0]?.[0].sourceId).toBe("OL24333986M");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et le prince de sang-mêlé",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
isbn13: "9782070612383",
|
||||
description: "Sixième année à Poudlard."
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("replaces an ambiguous title-only identification when a later provider supplies a described record", async () => {
|
||||
const database = createDatabase();
|
||||
database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'bnf'").run();
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "Rosemary Carter",
|
||||
isbn: "9780373105601",
|
||||
publisher: "Harlequin Books",
|
||||
publishedDate: "1982"
|
||||
}
|
||||
]
|
||||
};
|
||||
const bnfProvider: MetadataProvider = {
|
||||
id: "bnf",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "scénario, Roy Thomas, Gary Friedrich",
|
||||
isbn: "9782809476255",
|
||||
description: "Daredevil affronte l'Homme aux échasses.",
|
||||
language: "fre",
|
||||
publisher: "Panini comics",
|
||||
publishedDate: "2019"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
bnfProvider as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Daredevil",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
};
|
||||
|
||||
const result = await service.enrichMetadata(localMetadata, "/library/Daredevil.cbz", { remote: true });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: "Daredevil",
|
||||
author: "scénario, Roy Thomas, Gary Friedrich",
|
||||
isbn: "9782809476255",
|
||||
isbn13: "9782809476255",
|
||||
description: "Daredevil affronte l'Homme aux échasses."
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
});
|
||||
|
||||
function createDatabase(): DatabaseService {
|
||||
const dir = mkdtempSync(join(tmpdir(), "readabook-metadata-service-"));
|
||||
tempDirs.push(dir);
|
||||
process.env.DATABASE_PATH = join(dir, "readabook.sqlite");
|
||||
process.env.STORAGE_DIR = join(dir, "storage");
|
||||
return new DatabaseService();
|
||||
}
|
||||
|
||||
function providerStub(id: MetadataProvider["id"]): MetadataProvider {
|
||||
return {
|
||||
id,
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => []
|
||||
};
|
||||
}
|
||||
|
||||
function canLoadBetterSqlite(): boolean {
|
||||
try {
|
||||
const database = createDatabase();
|
||||
database.onModuleDestroy();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -11,13 +11,17 @@ import { BnfProvider } from "./adapters/bnf.provider.js";
|
||||
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
|
||||
import { LocalMetadataProvider } from "./adapters/local.provider.js";
|
||||
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
|
||||
import { MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js";
|
||||
import { BookIdentifiers, LocalMetadataHints, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js";
|
||||
import { ExtractIdentifiers, toIsbn13 } from "./use-cases/extract-identifiers.js";
|
||||
import { ExtractLocalMetadataHints } from "./use-cases/extract-local-metadata-hints.js";
|
||||
import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js";
|
||||
import { ScoreMetadataMatch } from "./use-cases/score-metadata-match.js";
|
||||
|
||||
@Injectable()
|
||||
export class MetadataService {
|
||||
private readonly extractIdentifiers = new ExtractIdentifiers();
|
||||
private readonly extractLocalMetadataHints = new ExtractLocalMetadataHints();
|
||||
private readonly scoreMetadataMatch = new ScoreMetadataMatch();
|
||||
private readonly resolveProviderChain: ResolveProviderChain;
|
||||
|
||||
constructor(
|
||||
@ -67,8 +71,13 @@ export class MetadataService {
|
||||
return this.getSourcesConfig();
|
||||
}
|
||||
|
||||
async enrichMetadata(localMetadata: BookMetadata, filePath: string, options: { remote: boolean }): Promise<BookMetadata & { isbn13: string | null; identifiersJson: string }> {
|
||||
async enrichMetadata(
|
||||
localMetadata: BookMetadata,
|
||||
filePath: string,
|
||||
options: { remote: boolean }
|
||||
): Promise<BookMetadata & { isbn13: string | null; identifiersJson: string; localMetadataJson: string }> {
|
||||
const identifiers = this.extractIdentifiers.fromMetadataAndFile(localMetadata, filePath);
|
||||
const local = this.extractLocalMetadataHints.fromMetadataAndFile(localMetadata, filePath);
|
||||
const configs = this.getProviderConfigs();
|
||||
const chain = options.remote
|
||||
? this.resolveProviderChain.resolve(configs)
|
||||
@ -77,16 +86,42 @@ export class MetadataService {
|
||||
|
||||
for (const { provider, config } of chain) {
|
||||
try {
|
||||
const match = await provider.lookup(
|
||||
{
|
||||
title: merged.title,
|
||||
author: merged.author,
|
||||
filePath,
|
||||
identifiers
|
||||
},
|
||||
config
|
||||
);
|
||||
if (match) merged = mergeMetadata(merged, match);
|
||||
const hasIsbn = Boolean(identifiers.isbn13 ?? identifiers.isbn10);
|
||||
const match =
|
||||
provider.id === "local" || hasIsbn
|
||||
? await provider.lookup(
|
||||
{
|
||||
title: merged.title,
|
||||
author: merged.author,
|
||||
filePath,
|
||||
sourceId: null,
|
||||
identifiers,
|
||||
local
|
||||
},
|
||||
config
|
||||
)
|
||||
: null;
|
||||
if (match) {
|
||||
merged = mergeMetadata(merged, match);
|
||||
continue;
|
||||
}
|
||||
if (!options.remote || provider.id === "local") continue;
|
||||
const query = {
|
||||
title: local.title,
|
||||
author: local.author,
|
||||
year: local.year,
|
||||
isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn
|
||||
};
|
||||
const best = this.scoreMetadataMatch.best(query, await provider.searchByMetadata(query, config));
|
||||
if (best) {
|
||||
if (!isActionableSearchMatch(best.match)) continue;
|
||||
merged = shouldReplaceAmbiguousIdentification(localMetadata, merged, best.match)
|
||||
? mergeMetadata({ ...localMetadata, author: null, isbn: null, description: null, language: null, publisher: null, publishedDate: null }, best.match)
|
||||
: mergeMetadata(merged, best.match);
|
||||
|
||||
const detailedMatch = await this.lookupSearchMatchDetails(provider, config, filePath, identifiers, local, merged, best.match);
|
||||
if (detailedMatch) merged = mergeMetadata(merged, detailedMatch);
|
||||
}
|
||||
} catch {
|
||||
// Provider failures must not block local ingestion.
|
||||
}
|
||||
@ -97,7 +132,8 @@ export class MetadataService {
|
||||
...merged,
|
||||
isbn: merged.isbn ?? isbn13 ?? identifiers.isbn10,
|
||||
isbn13,
|
||||
identifiersJson: JSON.stringify(identifiers)
|
||||
identifiersJson: JSON.stringify(identifiers),
|
||||
localMetadataJson: JSON.stringify(local)
|
||||
};
|
||||
}
|
||||
|
||||
@ -124,6 +160,7 @@ export class MetadataService {
|
||||
isbn: enriched.isbn,
|
||||
isbn13: enriched.isbn13,
|
||||
identifiersJson: enriched.identifiersJson,
|
||||
localMetadataJson: enriched.localMetadataJson,
|
||||
language: enriched.language,
|
||||
publisher: enriched.publisher,
|
||||
publishedDate: enriched.publishedDate,
|
||||
@ -151,6 +188,48 @@ export class MetadataService {
|
||||
private getAutomationRow(): typeof automationSettings.$inferSelect {
|
||||
return this.database.db.select().from(automationSettings).where(eq(automationSettings.id, 1)).get()!;
|
||||
}
|
||||
|
||||
private async lookupSearchMatchDetails(
|
||||
provider: MetadataProvider,
|
||||
config: MetadataProviderConfig,
|
||||
filePath: string,
|
||||
identifiers: BookIdentifiers,
|
||||
local: LocalMetadataHints,
|
||||
merged: BookMetadata,
|
||||
match: MetadataMatch
|
||||
): Promise<MetadataMatch | null> {
|
||||
const derivedIdentifiers = {
|
||||
...identifiers,
|
||||
isbn13: identifiers.isbn13 ?? (match.isbn ? toIsbn13(match.isbn) : null),
|
||||
isbn10: identifiers.isbn10 ?? match.isbn ?? null,
|
||||
candidates: [...new Set([...identifiers.candidates, ...(match.isbn ? [match.isbn] : [])])]
|
||||
};
|
||||
const hasNewIdentifier = derivedIdentifiers.isbn13 !== identifiers.isbn13 || derivedIdentifiers.isbn10 !== identifiers.isbn10;
|
||||
const hasLookupTarget = hasNewIdentifier || Boolean(match.sourceId);
|
||||
const needsDetails = !merged.description && hasLookupTarget;
|
||||
if (!hasLookupTarget && !needsDetails) return null;
|
||||
|
||||
return provider.lookup(
|
||||
{
|
||||
title: merged.title,
|
||||
author: merged.author,
|
||||
filePath,
|
||||
sourceId: match.sourceId,
|
||||
identifiers: derivedIdentifiers,
|
||||
local
|
||||
},
|
||||
config
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isActionableSearchMatch(match: MetadataMatch): boolean {
|
||||
return Boolean(match.isbn ?? match.sourceId ?? match.description);
|
||||
}
|
||||
|
||||
function shouldReplaceAmbiguousIdentification(local: BookMetadata, current: BookMetadata, next: MetadataMatch): boolean {
|
||||
if (local.author || local.isbn || !current.isbn || !next.isbn || current.isbn === next.isbn) return false;
|
||||
return Boolean(next.description);
|
||||
}
|
||||
|
||||
function mergeMetadata(current: BookMetadata, next: MetadataMatch): BookMetadata {
|
||||
|
||||
@ -12,10 +12,34 @@ export type MetadataLookup = {
|
||||
title: string;
|
||||
author: string | null;
|
||||
filePath: string;
|
||||
sourceId?: string | null;
|
||||
identifiers: BookIdentifiers;
|
||||
local: LocalMetadataHints;
|
||||
};
|
||||
|
||||
export type LocalMetadataHints = {
|
||||
title: string;
|
||||
author: string | null;
|
||||
year: string | null;
|
||||
isbn: string | null;
|
||||
fileTitle: string;
|
||||
raw: {
|
||||
title: string;
|
||||
author: string | null;
|
||||
publishedDate: string | null;
|
||||
fileName: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type MetadataSearchQuery = {
|
||||
title: string;
|
||||
author: string | null;
|
||||
year?: string | null;
|
||||
isbn?: string | null;
|
||||
};
|
||||
|
||||
export type MetadataMatch = Partial<BookMetadata> & {
|
||||
sourceId?: string | null;
|
||||
identifiers?: Partial<BookIdentifiers>;
|
||||
};
|
||||
|
||||
@ -29,4 +53,5 @@ export type MetadataProviderConfig = {
|
||||
export interface MetadataProvider {
|
||||
readonly id: MetadataProviderId;
|
||||
lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null>;
|
||||
searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]>;
|
||||
}
|
||||
|
||||
@ -4,7 +4,8 @@ import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js";
|
||||
|
||||
const provider = (id: MetadataProvider["id"]): MetadataProvider => ({
|
||||
id,
|
||||
lookup: async () => null
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => []
|
||||
});
|
||||
|
||||
describe("ResolveProviderChain", () => {
|
||||
|
||||
@ -39,7 +39,7 @@ export class ExtractIdentifiers {
|
||||
export function normalizeIsbn(value: string): string | null {
|
||||
const compact = value.replace(/[^0-9X]/gi, "").toUpperCase();
|
||||
if (compact.length === 10 && isValidIsbn10(compact)) return compact;
|
||||
if (compact.length === 13 && isValidIsbn13(compact)) return compact;
|
||||
if (compact.length === 13 && /^97[89]/.test(compact) && isValidIsbn13(compact)) return compact;
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,66 @@
|
||||
import { basename, extname } from "node:path";
|
||||
import { BookMetadata } from "../../scanner/metadata.js";
|
||||
import { LocalMetadataHints } from "../metadata.types.js";
|
||||
|
||||
export class ExtractLocalMetadataHints {
|
||||
fromMetadataAndFile(metadata: BookMetadata, filePath: string): LocalMetadataHints {
|
||||
const fileName = basename(filePath, extname(filePath));
|
||||
const parsed = parseFileName(fileName);
|
||||
const title = cleanTitle(metadata.title) || parsed.title || fileName;
|
||||
const author = cleanValue(metadata.author) ?? parsed.author;
|
||||
const year = yearFrom(metadata.publishedDate) ?? parsed.year;
|
||||
|
||||
return {
|
||||
title,
|
||||
author,
|
||||
year,
|
||||
isbn: cleanValue(metadata.isbn),
|
||||
fileTitle: parsed.title ?? fileName,
|
||||
raw: {
|
||||
title: metadata.title,
|
||||
author: metadata.author,
|
||||
publishedDate: metadata.publishedDate,
|
||||
fileName
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseFileName(fileName: string): { title: string | null; author: string | null; year: string | null } {
|
||||
let value = fileName.replace(/[_]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
const year = yearFrom(value);
|
||||
if (year) value = value.replace(new RegExp(`\\b${year}\\b`), " ");
|
||||
|
||||
const parenthetical = [...value.matchAll(/\(([^()]{2,120})\)/g)].map((match) => match[1].trim());
|
||||
const authorFromParentheses = parenthetical.find((item) => looksLikeAuthor(item)) ?? null;
|
||||
value = value.replace(/\([^()]*\)/g, " ");
|
||||
|
||||
const split = value.match(/^(.+?)\s+-\s+(.+)$/);
|
||||
const title = cleanTitle(split?.[1] ?? value);
|
||||
const author = cleanValue(split?.[2]) ?? authorFromParentheses;
|
||||
return { title, author, year };
|
||||
}
|
||||
|
||||
function cleanTitle(value: string | null): string | null {
|
||||
if (!value) return null;
|
||||
const cleaned = value
|
||||
.replace(/\[[^\]]*\]/g, " ")
|
||||
.replace(/\b(epub|pdf|retail|ebook|scan)\b/gi, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
return cleaned || null;
|
||||
}
|
||||
|
||||
function cleanValue(value: string | null | undefined): string | null {
|
||||
if (!value) return null;
|
||||
const cleaned = value.replace(/\s+/g, " ").trim();
|
||||
return cleaned || null;
|
||||
}
|
||||
|
||||
function yearFrom(value: string | null): string | null {
|
||||
return value?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function looksLikeAuthor(value: string): boolean {
|
||||
return /[A-Za-zÀ-ÖØ-öø-ÿ]/.test(value) && (value.includes(".") || value.includes(" ") || /^[A-Z][a-z]+$/.test(value));
|
||||
}
|
||||
58
apps/api/src/metadata/use-cases/score-metadata-match.ts
Normal file
58
apps/api/src/metadata/use-cases/score-metadata-match.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import { MetadataMatch, MetadataSearchQuery } from "../metadata.types.js";
|
||||
|
||||
export type ScoredMetadataMatch = {
|
||||
match: MetadataMatch;
|
||||
score: number;
|
||||
};
|
||||
|
||||
export class ScoreMetadataMatch {
|
||||
score(query: MetadataSearchQuery, match: MetadataMatch): number {
|
||||
let score = 0;
|
||||
const titleScore = similarity(normalize(query.title), normalize(match.title ?? ""));
|
||||
score += titleScore * 0.7;
|
||||
|
||||
if (query.author && match.author) {
|
||||
score += similarity(normalize(query.author), normalize(match.author)) * 0.2;
|
||||
} else if (!query.author) {
|
||||
score += 0.08;
|
||||
}
|
||||
|
||||
const queryYear = query.year ?? null;
|
||||
const matchYear = match.publishedDate?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
if (queryYear && matchYear) score += queryYear === matchYear ? 0.1 : -0.1;
|
||||
|
||||
if (query.isbn && match.isbn && query.isbn.replace(/\D/g, "") === match.isbn.replace(/\D/g, "")) {
|
||||
score += 0.25;
|
||||
}
|
||||
return Math.max(0, Math.min(1, score));
|
||||
}
|
||||
|
||||
best(query: MetadataSearchQuery, matches: MetadataMatch[], minimumScore = 0.55): ScoredMetadataMatch | null {
|
||||
const scored = matches
|
||||
.map((match) => ({ match, score: this.score(query, match) }))
|
||||
.sort((left, right) => right.score - left.score);
|
||||
const best = scored[0];
|
||||
return best && best.score >= minimumScore ? best : null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(value: string): string {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.replace(/\b(le|la|les|the|a|an|de|du|des|et|and)\b/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function similarity(left: string, right: string): number {
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
const leftTokens = new Set(left.split(" "));
|
||||
const rightTokens = new Set(right.split(" "));
|
||||
const intersection = [...leftTokens].filter((token) => rightTokens.has(token)).length;
|
||||
const union = new Set([...leftTokens, ...rightTokens]).size;
|
||||
return union ? intersection / union : 0;
|
||||
}
|
||||
@ -109,6 +109,7 @@ export class ScannerService {
|
||||
isbn: metadata.isbn,
|
||||
isbn13: metadata.isbn13,
|
||||
identifiersJson: metadata.identifiersJson,
|
||||
localMetadataJson: metadata.localMetadataJson,
|
||||
language: metadata.language,
|
||||
publisher: metadata.publisher,
|
||||
publishedDate: metadata.publishedDate,
|
||||
@ -137,6 +138,19 @@ export class ScannerService {
|
||||
isbn: null,
|
||||
isbn13: null,
|
||||
identifiersJson: JSON.stringify({ candidates: [], isbn10: null, isbn13: null }),
|
||||
localMetadataJson: JSON.stringify({
|
||||
title: basename(filePath, extname(filePath)),
|
||||
author: null,
|
||||
year: null,
|
||||
isbn: null,
|
||||
fileTitle: basename(filePath, extname(filePath)),
|
||||
raw: {
|
||||
title: basename(filePath, extname(filePath)),
|
||||
author: null,
|
||||
publishedDate: null,
|
||||
fileName: basename(filePath, extname(filePath))
|
||||
}
|
||||
}),
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
|
||||
Reference in New Issue
Block a user