feat(api,web): providers ComicVine + MangaDex et pilotage de l'enrichissement
- Adaptateurs comic-vine et mangadex avec helper de fetch partagé, timeouts et fallback durcis sur les providers existants - Scoring des correspondances amélioré, normalisation de la date de publication, chaîne de résolution des providers étendue - Scanner : statuts par livre (scan/enrichissement) et page admin d'automatisation alignée Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Git Agent
@ -2,6 +2,8 @@ import { Injectable } from "@nestjs/common";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch, providerHttpError } from "./provider-fetch.js";
|
||||
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true });
|
||||
|
||||
@ -36,8 +38,8 @@ export class BnfProvider implements MetadataProvider {
|
||||
url.searchParams.set("query", query);
|
||||
url.searchParams.set("maximumRecords", String(maximumRecords));
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
||||
if (!response.ok) return [];
|
||||
const response = await providerFetch(this.id, url, { timeoutMs: 5000 });
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `BnF HTTP ${response.status}`);
|
||||
const parsed = parser.parse(await response.text());
|
||||
const records = asArray(parsed?.searchRetrieveResponse?.records?.record)
|
||||
.map((entry) => (entry.recordData as Record<string, unknown> | undefined)?.record)
|
||||
@ -53,7 +55,7 @@ export class BnfProvider implements MetadataProvider {
|
||||
isbn: bestIsbn(fields, expectedIsbn13),
|
||||
language: subfield(fields, "101", "a"),
|
||||
publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"),
|
||||
publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))
|
||||
publishedDate: normalizePublishedDate(cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d")))
|
||||
};
|
||||
})
|
||||
.sort((left, right) => Number(Boolean(right.isbn)) - Number(Boolean(left.isbn)));
|
||||
|
||||
160
apps/api/src/metadata/adapters/comic-vine.provider.ts
Normal file
160
apps/api/src/metadata/adapters/comic-vine.provider.ts
Normal file
@ -0,0 +1,160 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class ComicVineProvider implements MetadataProvider {
|
||||
readonly id = "comicvine" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
assertApiKey(config);
|
||||
const title = cleanComicTitle(query.title);
|
||||
const relaxed = title.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim();
|
||||
const matches = [
|
||||
...(await searchComicVine("volume", title, config)),
|
||||
...(await searchComicVine("issue", title, config)),
|
||||
...(relaxed && relaxed !== title ? await searchComicVine("volume", relaxed, config) : [])
|
||||
];
|
||||
return rankMatches(query.title, dedupe(matches));
|
||||
}
|
||||
}
|
||||
|
||||
export class ComicVineProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "missing-key" | "invalid-key" | "rate-limit" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ComicVineProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function searchComicVine(resource: "volume" | "issue", title: string, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://comicvine.gamespot.com/api/search/");
|
||||
url.searchParams.set("api_key", config.apiKey!);
|
||||
url.searchParams.set("format", "json");
|
||||
url.searchParams.set("resources", resource);
|
||||
url.searchParams.set("query", title);
|
||||
url.searchParams.set("limit", "10");
|
||||
url.searchParams.set(
|
||||
"field_list",
|
||||
resource === "volume" ? "id,name,description,image,start_year,publisher" : "id,name,description,image,cover_date,store_date,volume"
|
||||
);
|
||||
const response = await providerFetch("comicvine", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (Comic Vine; non-commercial)" },
|
||||
timeoutMs: 6000
|
||||
});
|
||||
const data = (await parseComicVineResponse(response)) as { results?: Array<Record<string, unknown>> };
|
||||
return (data.results ?? []).map((entry) => comicVineToMatch(resource, entry));
|
||||
}
|
||||
|
||||
function assertApiKey(config: MetadataProviderConfig): void {
|
||||
if (!config.apiKey?.trim()) {
|
||||
throw new ComicVineProviderError("missing-key", 0, "Comic Vine API key is required");
|
||||
}
|
||||
}
|
||||
|
||||
async function parseComicVineResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { status_code?: number; error?: string };
|
||||
if (response.status === 429) throw new ComicVineProviderError("rate-limit", response.status, data.error ?? "Comic Vine rate limit");
|
||||
if (response.status === 401 || response.status === 403) throw new ComicVineProviderError("invalid-key", response.status, data.error ?? "Comic Vine API key rejected");
|
||||
if (!response.ok) throw new ComicVineProviderError("http", response.status, data.error ?? `Comic Vine HTTP ${response.status}`);
|
||||
if (data.status_code && data.status_code !== 1) {
|
||||
const code = data.status_code === 100 || data.status_code === 101 ? "invalid-key" : "http";
|
||||
throw new ComicVineProviderError(code, 200, data.error ?? `Comic Vine status ${data.status_code}`);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function comicVineToMatch(resource: "volume" | "issue", entry: Record<string, unknown>): MetadataMatch & { comicVineRank?: number } {
|
||||
const volume = objectValue(entry.volume);
|
||||
const title = resource === "issue" ? [stringValue(volume.name), stringValue(entry.name)].filter(Boolean).join(" ") : stringValue(entry.name);
|
||||
return {
|
||||
title: title || undefined,
|
||||
description: cleanHtml(stringValue(entry.description)),
|
||||
publisher: stringValue(objectValue(entry.publisher).name),
|
||||
publishedDate: normalizePublishedDate(resource === "volume" ? stringValue(entry.start_year) : yearFromDate(stringValue(entry.cover_date) ?? stringValue(entry.store_date))),
|
||||
coverUrl: imageUrl(entry.image),
|
||||
sourceId: stringValue(entry.id)
|
||||
};
|
||||
}
|
||||
|
||||
function cleanComicTitle(value: string): string {
|
||||
return value
|
||||
.replace(/\.[A-Za-z0-9]{2,5}$/g, " ")
|
||||
.replace(/[._]+/g, " ")
|
||||
.replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ")
|
||||
.replace(/\([^)]*\)/g, " ")
|
||||
.replace(/\b(e?bdz|Paprika\+?|emuleCenter(?:\.|\s+)net)\b/gi, " ")
|
||||
.replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/[+]+/g, " ")
|
||||
.replace(/\s+-\s+/g, " ")
|
||||
.replace(/\s*-\s*$/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function rankMatches(originalTitle: string, matches: Array<MetadataMatch & { comicVineRank?: number }>): MetadataMatch[] {
|
||||
return [...matches]
|
||||
.map((match) => ({ ...match, comicVineRank: comicRank(originalTitle, match) }))
|
||||
.sort((left, right) => (right.comicVineRank ?? 0) - (left.comicVineRank ?? 0))
|
||||
.map(({ comicVineRank: _rank, ...match }) => match);
|
||||
}
|
||||
|
||||
function comicRank(originalTitle: string, match: MetadataMatch): number {
|
||||
let rank = tokenOverlap(cleanComicTitle(originalTitle), match.title ?? "") * 10;
|
||||
if (match.coverUrl) rank += 1;
|
||||
if (match.description) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function dedupe(matches: MetadataMatch[]): MetadataMatch[] {
|
||||
const seen = new Set<string>();
|
||||
return matches.filter((match) => {
|
||||
const key = [match.sourceId, match.title].filter(Boolean).join("|");
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function cleanHtml(value: string | null): string | null {
|
||||
if (!value) return null;
|
||||
return value.replace(/<[^>]*>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/\s+/g, " ").trim() || null;
|
||||
}
|
||||
|
||||
function imageUrl(value: unknown): string | null {
|
||||
const image = objectValue(value);
|
||||
return stringValue(image.original_url) ?? stringValue(image.super_url) ?? stringValue(image.medium_url) ?? stringValue(image.small_url);
|
||||
}
|
||||
|
||||
function yearFromDate(value: string | null): string | null {
|
||||
return value?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTokens(left));
|
||||
const rightTokens = new Set(normalizeTokens(right));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTokens(value: string): string[] {
|
||||
return value.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(" ").filter(Boolean);
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
if (typeof value === "number") return String(value);
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class GoogleBooksProvider implements MetadataProvider {
|
||||
@ -12,54 +14,157 @@ export class GoogleBooksProvider implements MetadataProvider {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
const query = `isbn:${isbn}`;
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("maxResults", "1");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
if (!response.ok) return null;
|
||||
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
|
||||
const info = data.items?.[0]?.volumeInfo;
|
||||
if (!info) return null;
|
||||
return {
|
||||
title: stringValue(info.title) ?? undefined,
|
||||
author: arrayJoin(info.authors),
|
||||
description: stringValue(info.description),
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13)
|
||||
};
|
||||
const matches = await this.searchVolumes(`isbn:${isbn}`, config, lookup.title, lookup.author, lookup.identifiers.isbn13);
|
||||
return rankMatches(lookup.title, matches)[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const attempts = googleBookQueries(query);
|
||||
const matches: MetadataMatch[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const attempt of attempts) {
|
||||
for (const match of await this.searchVolumes(attempt, config, query.title, query.author, query.isbn ? toIsbn13(query.isbn) : null)) {
|
||||
const key = [match.sourceId, match.isbn, match.title, match.author].filter(Boolean).join("|");
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
matches.push(match);
|
||||
}
|
||||
}
|
||||
return rankMatches(query.title, matches);
|
||||
}
|
||||
|
||||
private async searchVolumes(
|
||||
googleQuery: string,
|
||||
config: MetadataProviderConfig,
|
||||
originalTitle: string,
|
||||
originalAuthor: string | null,
|
||||
expectedIsbn13: string | null
|
||||
): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", `intitle:${query.title}${query.author ? `+inauthor:${query.author}` : ""}`);
|
||||
url.searchParams.set("maxResults", "5");
|
||||
url.searchParams.set("q", googleQuery);
|
||||
url.searchParams.set("maxResults", "10");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
if (!response.ok) return [];
|
||||
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
|
||||
const response = await providerFetch(this.id, url, { timeoutMs: 4000 });
|
||||
const data = (await parseGoogleResponse(response)) as { items?: Array<{ id?: string; volumeInfo?: Record<string, unknown> }> };
|
||||
return (data.items ?? [])
|
||||
.map((item) => item.volumeInfo)
|
||||
.filter((info): info is Record<string, unknown> => Boolean(info))
|
||||
.map((info) => ({
|
||||
.map((item) => ({ sourceId: item.id, info: item.volumeInfo }))
|
||||
.filter((item): item is { sourceId: string | undefined; info: Record<string, unknown> } => Boolean(item.info))
|
||||
.map(({ sourceId, info }) => ({
|
||||
title: stringValue(info.title) ?? undefined,
|
||||
author: arrayJoin(info.authors),
|
||||
description: stringValue(info.description),
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
publishedDate: normalizePublishedDate(stringValue(info.publishedDate)),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, expectedIsbn13),
|
||||
coverUrl: coverUrl(info.imageLinks),
|
||||
sourceId,
|
||||
identifiers: { candidates: isbnCandidates(info.industryIdentifiers) },
|
||||
googleRank: googleRank(originalTitle, originalAuthor, info)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
export class GoogleBooksProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "quota" | "auth" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "GoogleBooksProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function parseGoogleResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { error?: { message?: string; status?: string } };
|
||||
if (response.ok) return data;
|
||||
const message = data.error?.message ?? `Google Books HTTP ${response.status}`;
|
||||
if (response.status === 429) throw new GoogleBooksProviderError("quota", response.status, message);
|
||||
if (response.status === 401 || response.status === 403) throw new GoogleBooksProviderError("auth", response.status, message);
|
||||
throw new GoogleBooksProviderError("http", response.status, message);
|
||||
}
|
||||
|
||||
function googleBookQueries(query: MetadataSearchQuery): string[] {
|
||||
const cleaned = cleanGoogleBooksTitle(query.title);
|
||||
const relaxed = relaxSeriesTitle(cleaned);
|
||||
return [
|
||||
query.isbn ? `isbn:${query.isbn}` : null,
|
||||
googleTitleQuery(cleaned, query.author, true),
|
||||
googleTitleQuery(cleaned, query.author, false),
|
||||
relaxed !== cleaned ? googleTitleQuery(relaxed, query.author, true) : null,
|
||||
relaxed !== cleaned ? googleTitleQuery(relaxed, null, false) : null,
|
||||
googleTitleQuery(cleaned, null, false)
|
||||
].filter((value, index, values): value is string => Boolean(value) && values.indexOf(value) === index);
|
||||
}
|
||||
|
||||
function googleTitleQuery(title: string, author: string | null, quoted: boolean): string {
|
||||
const titlePart = quoted ? `intitle:"${title.replace(/"/g, " ")}"` : `intitle:${title}`;
|
||||
return author ? `${titlePart}+inauthor:${author}` : titlePart;
|
||||
}
|
||||
|
||||
function cleanGoogleBooksTitle(value: string): string {
|
||||
return value
|
||||
.replace(/\.[A-Za-z0-9]{2,5}$/g, " ")
|
||||
.replace(/[._]+/g, " ")
|
||||
.replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ")
|
||||
.replace(/\b(e?bdz|Paprika\+?|emuleCenter\.net)\b/gi, " ")
|
||||
.replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/\bVol(?:ume)?\.?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/[+]+/g, " ")
|
||||
.replace(/\s+-\s+/g, " ")
|
||||
.replace(/\s*-\s*$/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function relaxSeriesTitle(value: string): string {
|
||||
return value.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function rankMatches(originalTitle: string, matches: Array<MetadataMatch & { googleRank?: number }>): MetadataMatch[] {
|
||||
return [...matches]
|
||||
.sort((left, right) => (right.googleRank ?? 0) - (left.googleRank ?? 0))
|
||||
.map(({ googleRank: _googleRank, ...match }) => match);
|
||||
}
|
||||
|
||||
function googleRank(originalTitle: string, originalAuthor: string | null, info: Record<string, unknown>): number {
|
||||
const expectedVolume = volumeNumber(originalTitle);
|
||||
const candidateTitle = [stringValue(info.title), stringValue(info.subtitle)].filter(Boolean).join(" ");
|
||||
let rank = tokenOverlap(cleanGoogleBooksTitle(originalTitle), candidateTitle) * 10;
|
||||
if (expectedVolume) {
|
||||
const candidateVolume = volumeNumber(candidateTitle);
|
||||
rank += candidateVolume === expectedVolume ? 6 : candidateVolume ? -4 : 0;
|
||||
}
|
||||
if (originalAuthor && arrayJoin(info.authors)?.toLowerCase().includes(originalAuthor.toLowerCase())) rank += 2;
|
||||
if (stringValue(info.description)) rank += 1;
|
||||
if (coverUrl(info.imageLinks)) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function volumeNumber(value: string): string | null {
|
||||
return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTokens(left));
|
||||
const rightTokens = new Set(normalizeTokens(right));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTokens(value: string): string[] {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.split(" ")
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -77,3 +182,20 @@ function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | nu
|
||||
const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier;
|
||||
return stringValue(isbn13) ?? stringValue(isbn10);
|
||||
}
|
||||
|
||||
function isbnCandidates(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map((entry) => stringValue((entry as { identifier?: unknown }).identifier)).filter((entry): entry is string => Boolean(entry));
|
||||
}
|
||||
|
||||
function coverUrl(value: unknown): string | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const links = value as Record<string, unknown>;
|
||||
return (
|
||||
stringValue(links.extraLarge) ??
|
||||
stringValue(links.large) ??
|
||||
stringValue(links.medium) ??
|
||||
stringValue(links.thumbnail) ??
|
||||
stringValue(links.smallThumbnail)
|
||||
)?.replace(/^http:/, "https:") ?? null;
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
|
||||
@Injectable()
|
||||
export class LocalMetadataProvider implements MetadataProvider {
|
||||
@ -20,7 +21,7 @@ export class LocalMetadataProvider implements MetadataProvider {
|
||||
title: query.title,
|
||||
author: query.author,
|
||||
isbn: query.isbn ?? null,
|
||||
publishedDate: query.year ?? null
|
||||
publishedDate: normalizePublishedDate(query.year)
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
179
apps/api/src/metadata/adapters/mangadex.provider.ts
Normal file
179
apps/api/src/metadata/adapters/mangadex.provider.ts
Normal file
@ -0,0 +1,179 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { extractSeriesVolume } from "../use-cases/extract-series-volume.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class MangaDexProvider implements MetadataProvider {
|
||||
readonly id = "mangadex" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const titles = mangaDexTitleQueries(query.title);
|
||||
const matches: Array<MetadataMatch & { mangaDexRank?: number }> = [];
|
||||
const seen = new Map<string, number>();
|
||||
for (const title of titles) {
|
||||
for (const match of await searchManga(title, config)) {
|
||||
const key = match.sourceId ?? `${match.title}|${match.author}`;
|
||||
const ranked = { ...match, scoreTitle: title, mangaDexRank: mangaRank(query.title, title, match) };
|
||||
const existingIndex = seen.get(key);
|
||||
if (existingIndex === undefined) {
|
||||
seen.set(key, matches.length);
|
||||
matches.push(ranked);
|
||||
continue;
|
||||
}
|
||||
if ((ranked.mangaDexRank ?? 0) > (matches[existingIndex]?.mangaDexRank ?? 0)) {
|
||||
matches[existingIndex] = ranked;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rankMatches(matches);
|
||||
}
|
||||
}
|
||||
|
||||
export class MangaDexProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "rate-limit" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "MangaDexProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function searchManga(title: string, _config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://api.mangadex.org/manga");
|
||||
url.searchParams.set("title", title);
|
||||
url.searchParams.set("limit", "10");
|
||||
url.searchParams.set("includes[]", "cover_art");
|
||||
url.searchParams.append("includes[]", "author");
|
||||
url.searchParams.append("includes[]", "artist");
|
||||
url.searchParams.set("contentRating[]", "safe");
|
||||
url.searchParams.append("contentRating[]", "suggestive");
|
||||
let response = await providerFetch("mangadex", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" },
|
||||
timeoutMs: 5000
|
||||
});
|
||||
if (response.status === 429) {
|
||||
await sleep(retryDelayMs(response));
|
||||
response = await providerFetch("mangadex", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" },
|
||||
timeoutMs: 5000
|
||||
});
|
||||
}
|
||||
const data = (await parseMangaDexResponse(response)) as { data?: Array<Record<string, unknown>> };
|
||||
return (data.data ?? []).map(mangaToMatch);
|
||||
}
|
||||
|
||||
function retryDelayMs(response: Response): number {
|
||||
const retryAfter = Number(response.headers.get("Retry-After"));
|
||||
return Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter * 1000, 2000) : 250;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function parseMangaDexResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { errors?: Array<{ detail?: string; title?: string }> };
|
||||
if (response.ok) return data;
|
||||
const message = data.errors?.map((error) => error.detail ?? error.title).filter(Boolean).join("; ") || `MangaDex HTTP ${response.status}`;
|
||||
if (response.status === 429) throw new MangaDexProviderError("rate-limit", response.status, message);
|
||||
throw new MangaDexProviderError("http", response.status, message);
|
||||
}
|
||||
|
||||
function mangaToMatch(manga: Record<string, unknown>): MetadataMatch & { mangaDexRank?: number } {
|
||||
const id = stringValue(manga.id);
|
||||
const attributes = objectValue(manga.attributes);
|
||||
const relationships = Array.isArray(manga.relationships) ? (manga.relationships as Array<Record<string, unknown>>) : [];
|
||||
const cover = relationships.find((entry) => entry.type === "cover_art");
|
||||
const coverFile = stringValue(objectValue(cover?.attributes).fileName);
|
||||
return {
|
||||
title: localizedText(attributes.title) ?? undefined,
|
||||
author: relationshipNames(relationships),
|
||||
description: localizedText(attributes.description),
|
||||
publishedDate: normalizePublishedDate(stringValue(attributes.year)),
|
||||
language: stringValue(attributes.originalLanguage),
|
||||
coverUrl: id && coverFile ? `https://uploads.mangadex.org/covers/${id}/${coverFile}.512.jpg` : null,
|
||||
sourceId: id
|
||||
};
|
||||
}
|
||||
|
||||
function mangaDexTitleQueries(title: string): string[] {
|
||||
const cleaned = extractSeriesVolume(title).seriesTitle;
|
||||
return [cleaned, ...mangaDexTitleAliases(cleaned)].filter(
|
||||
(value, index, values): value is string => Boolean(value) && values.indexOf(value) === index
|
||||
);
|
||||
}
|
||||
|
||||
function mangaDexTitleAliases(title: string): string[] {
|
||||
const normalized = normalizeTitle(title);
|
||||
if (normalized === "demon slayer school days") return ["Demon Slayer Kimetsu Academy", "Kimetsu Academy"];
|
||||
return [];
|
||||
}
|
||||
|
||||
function mangaRank(originalTitle: string, searchedTitle: string, match: MetadataMatch): number {
|
||||
const expectedVolume = volumeNumber(originalTitle);
|
||||
let rank = tokenOverlap(searchedTitle, match.title ?? "") * 10;
|
||||
if (expectedVolume) {
|
||||
const candidateVolume = volumeNumber(match.title ?? "");
|
||||
rank += candidateVolume === expectedVolume ? 4 : candidateVolume ? -2 : 0;
|
||||
}
|
||||
if (match.coverUrl) rank += 1;
|
||||
if (match.description) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function rankMatches(matches: Array<MetadataMatch & { mangaDexRank?: number }>): MetadataMatch[] {
|
||||
return [...matches].sort((left, right) => (right.mangaDexRank ?? 0) - (left.mangaDexRank ?? 0)).map(({ mangaDexRank: _rank, ...match }) => match);
|
||||
}
|
||||
|
||||
function volumeNumber(value: string): string | null {
|
||||
return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTitle(left).split(" ").filter(Boolean));
|
||||
const rightTokens = new Set(normalizeTitle(right).split(" ").filter(Boolean));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function localizedText(value: unknown): string | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const entries = value as Record<string, unknown>;
|
||||
return stringValue(entries.en) ?? stringValue(entries.fr) ?? Object.values(entries).map(stringValue).find(Boolean) ?? null;
|
||||
}
|
||||
|
||||
function relationshipNames(relationships: Array<Record<string, unknown>>): string | null {
|
||||
const names = relationships
|
||||
.filter((entry) => entry.type === "author" || entry.type === "artist")
|
||||
.map((entry) => stringValue(objectValue(entry.attributes).name))
|
||||
.filter((entry): entry is string => Boolean(entry));
|
||||
return names.length ? [...new Set(names)].join(", ") : null;
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
if (typeof value === "number") return String(value);
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch, providerHttpError } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class OpenLibraryProvider implements MetadataProvider {
|
||||
@ -24,11 +26,11 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
if (query.author) url.searchParams.set("author", query.author);
|
||||
if (query.year) url.searchParams.set("first_publish_year", query.year);
|
||||
url.searchParams.set("limit", "5");
|
||||
const response = await fetch(url, {
|
||||
const response = await providerFetch(this.id, url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return [];
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
const data = (await response.json()) as { docs?: Array<Record<string, unknown>> };
|
||||
return (data.docs ?? []).map((doc) => ({
|
||||
title: stringValue(doc.title) ?? undefined,
|
||||
@ -36,28 +38,31 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
author: arrayJoin(doc.author_name),
|
||||
language: firstArrayValue(doc.language),
|
||||
publisher: firstArrayValue(doc.publisher),
|
||||
publishedDate: String(doc.first_publish_year ?? "") || null,
|
||||
isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
publishedDate: normalizePublishedDate(String(doc.first_publish_year ?? "") || null),
|
||||
isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null),
|
||||
coverUrl: openLibraryCoverUrl(doc.cover_i, firstArrayValue(doc.edition_key) ?? stringValue(doc.cover_edition_key))
|
||||
}));
|
||||
}
|
||||
|
||||
private async lookupEdition(sourceId: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const editionKey = sourceId.replace(/^\/?books\//, "");
|
||||
if (!editionKey) return null;
|
||||
const response = await fetch(`https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const response = await fetch(`https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
@ -70,18 +75,20 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
isbn: bestIsbn([...(asStringArray(edition.isbn_13)), ...(asStringArray(edition.isbn_10))], expectedIsbn13),
|
||||
language: languageValue(edition.languages),
|
||||
publisher: firstArrayValue(edition.publishers),
|
||||
publishedDate: stringValue(edition.publish_date)
|
||||
publishedDate: normalizePublishedDate(stringValue(edition.publish_date)),
|
||||
coverUrl: editionCoverUrl(edition)
|
||||
};
|
||||
}
|
||||
|
||||
private async lookupAuthorName(value: unknown): Promise<string | null> {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
if (typeof key !== "string") return null;
|
||||
const response = await fetch(`https://openlibrary.org${key}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org${key}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(3000)
|
||||
timeoutMs: 3000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
const author = (await response.json()) as Record<string, unknown>;
|
||||
return stringValue(author.name);
|
||||
}
|
||||
@ -124,3 +131,18 @@ function languageValue(value: unknown): string | null {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
return typeof key === "string" ? key.split("/").pop() ?? null : null;
|
||||
}
|
||||
|
||||
function openLibraryCoverUrl(coverId: unknown, editionKey: string | null): string | null {
|
||||
if (typeof coverId === "number" || typeof coverId === "string") {
|
||||
return `https://covers.openlibrary.org/b/id/${encodeURIComponent(String(coverId))}-L.jpg`;
|
||||
}
|
||||
if (editionKey) {
|
||||
return `https://covers.openlibrary.org/b/olid/${encodeURIComponent(editionKey)}-L.jpg`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function editionCoverUrl(edition: Record<string, unknown>): string | null {
|
||||
const covers = Array.isArray(edition.covers) ? edition.covers : [];
|
||||
return openLibraryCoverUrl(covers[0], stringValue(edition.key)?.split("/").pop() ?? null);
|
||||
}
|
||||
|
||||
88
apps/api/src/metadata/adapters/provider-fetch.ts
Normal file
88
apps/api/src/metadata/adapters/provider-fetch.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import { MetadataProviderId } from "../metadata.types.js";
|
||||
|
||||
export type MetadataProviderFailureCode = "timeout" | "dns" | "quota" | "auth" | "http" | "network";
|
||||
|
||||
export class MetadataProviderRequestError extends Error {
|
||||
constructor(
|
||||
readonly provider: MetadataProviderId | "cover",
|
||||
readonly code: MetadataProviderFailureCode,
|
||||
readonly message: string,
|
||||
readonly status?: number
|
||||
) {
|
||||
super(message);
|
||||
this.name = "MetadataProviderRequestError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function providerFetch(
|
||||
provider: MetadataProviderId | "cover",
|
||||
input: string | URL,
|
||||
init: RequestInit & { timeoutMs: number }
|
||||
): Promise<Response> {
|
||||
const { timeoutMs, ...requestInit } = init;
|
||||
try {
|
||||
return await fetch(input, {
|
||||
...requestInit,
|
||||
signal: requestInit.signal ?? AbortSignal.timeout(timeoutMs)
|
||||
});
|
||||
} catch (error) {
|
||||
throw classifyFetchError(provider, error, timeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
export async function providerHttpError(
|
||||
provider: MetadataProviderId | "cover",
|
||||
response: Response,
|
||||
fallbackMessage: string
|
||||
): Promise<MetadataProviderRequestError> {
|
||||
const message = (await response.text().catch(() => "")) || fallbackMessage;
|
||||
if (response.status === 429) return new MetadataProviderRequestError(provider, "quota", message, response.status);
|
||||
if (response.status === 401 || response.status === 403) return new MetadataProviderRequestError(provider, "auth", message, response.status);
|
||||
return new MetadataProviderRequestError(provider, "http", message, response.status);
|
||||
}
|
||||
|
||||
export function describeMetadataProviderError(error: unknown): string {
|
||||
if (error instanceof MetadataProviderRequestError) {
|
||||
const status = error.status ? ` HTTP ${error.status}` : "";
|
||||
return `${error.code}${status}: ${error.message}`;
|
||||
}
|
||||
if (hasProviderErrorCode(error)) {
|
||||
const status = typeof error.status === "number" ? ` HTTP ${error.status}` : "";
|
||||
return `${String(error.code)}${status}: ${errorMessage(error)}`;
|
||||
}
|
||||
return errorMessage(error);
|
||||
}
|
||||
|
||||
function classifyFetchError(provider: MetadataProviderId | "cover", error: unknown, timeoutMs: number): MetadataProviderRequestError {
|
||||
const code = nestedCode(error);
|
||||
if (isTimeoutError(error)) {
|
||||
return new MetadataProviderRequestError(provider, "timeout", `request timed out after ${timeoutMs}ms`);
|
||||
}
|
||||
if (code === "EAI_AGAIN" || code === "ENOTFOUND") {
|
||||
return new MetadataProviderRequestError(provider, "dns", code);
|
||||
}
|
||||
return new MetadataProviderRequestError(provider, "network", errorMessage(error));
|
||||
}
|
||||
|
||||
function isTimeoutError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError") ||
|
||||
error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")
|
||||
);
|
||||
}
|
||||
|
||||
function nestedCode(error: unknown): string | null {
|
||||
if (!error || typeof error !== "object") return null;
|
||||
const direct = "code" in error && typeof error.code === "string" ? error.code : null;
|
||||
if (direct) return direct;
|
||||
const cause = "cause" in error ? error.cause : null;
|
||||
return cause && typeof cause === "object" && "code" in cause && typeof cause.code === "string" ? cause.code : null;
|
||||
}
|
||||
|
||||
function hasProviderErrorCode(error: unknown): error is { code: string; status?: number; message?: string } {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && typeof error.code === "string");
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@ -38,6 +38,76 @@ describe("metadata match scoring", () => {
|
||||
);
|
||||
|
||||
expect(best?.match.title).toBe("Harry Potter et le Prince de sang-mêlé");
|
||||
expect(best?.score).toBeGreaterThan(0.7);
|
||||
expect(best?.score).toBeGreaterThan(70);
|
||||
});
|
||||
|
||||
it("scores titles, authors and dates with the contract weights", () => {
|
||||
const scorer = new ScoreMetadataMatch();
|
||||
const result = scorer.details(
|
||||
{ title: "The Harry Potter et le prince de sang mêlé: édition collector", author: "J. K. Rowling", year: "2005" },
|
||||
{
|
||||
title: "Harry Potter et le prince de sang-mêlé",
|
||||
author: "J.K. Rowling",
|
||||
publishedDate: "2006"
|
||||
}
|
||||
);
|
||||
|
||||
expect(result.titleScore).toBe(100);
|
||||
expect(result.authorScore).toBe(30);
|
||||
expect(result.dateScore).toBe(5);
|
||||
expect(result.score).toBe(96);
|
||||
});
|
||||
|
||||
it("uses exact ISBN matches before weaker title-only candidates", () => {
|
||||
const best = new ScoreMetadataMatch().best(
|
||||
{ title: "Daredevil", author: null, isbn: "9782809476255" },
|
||||
[
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "Rosemary Carter",
|
||||
isbn: "9780373105601"
|
||||
},
|
||||
{
|
||||
title: "Daredevil by Chip Zdarsky",
|
||||
author: "Chip Zdarsky",
|
||||
isbn: "9782809476255"
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
expect(best?.match.author).toBe("Chip Zdarsky");
|
||||
expect(best?.isbnMatch).toBe(true);
|
||||
});
|
||||
|
||||
it("scores unrelated serialized or audiobook candidates from title/author/date only", () => {
|
||||
const scorer = new ScoreMetadataMatch();
|
||||
const query = { title: "Harry Potter et le prince de sang mêlé", author: "J. K. Rowling" };
|
||||
const french = scorer.score(query, {
|
||||
title: "Harry Potter et le prince de sang-mêlé",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070577644"
|
||||
});
|
||||
const koreanVolume = scorer.score(query, {
|
||||
title: "Harry Potter et le prince de sang-mêlé - Volume 1",
|
||||
author: "J. K. Rowling",
|
||||
publisher: "문학수첩",
|
||||
isbn: "9791193790724"
|
||||
});
|
||||
|
||||
expect(french).toBeGreaterThan(80);
|
||||
expect(koreanVolume).toBeLessThan(french);
|
||||
});
|
||||
|
||||
it("does not apply legacy audiobook penalties outside the contract", () => {
|
||||
const scorer = new ScoreMetadataMatch();
|
||||
const query = { title: "Harry Potter et le prince de sang mêlé", author: "J. K. Rowling" };
|
||||
|
||||
expect(
|
||||
scorer.score(query, {
|
||||
title: "Harry Potter Et Le Prince De Sang-mêlé Livre Audio",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782075105170"
|
||||
})
|
||||
).toBeGreaterThan(80);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,7 +1,10 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BnfProvider } from "./adapters/bnf.provider.js";
|
||||
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
|
||||
import { ComicVineProvider } from "./adapters/comic-vine.provider.js";
|
||||
import { GoogleBooksProvider, GoogleBooksProviderError } from "./adapters/google-books.provider.js";
|
||||
import { MangaDexProvider } from "./adapters/mangadex.provider.js";
|
||||
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
|
||||
import { MetadataProviderRequestError } from "./adapters/provider-fetch.js";
|
||||
import { MetadataLookup, MetadataProviderConfig } from "./metadata.types.js";
|
||||
|
||||
const lookup: MetadataLookup = {
|
||||
@ -58,14 +61,210 @@ describe("metadata providers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("treats Google Books quota exhaustion as a non-blocking miss", async () => {
|
||||
it("reports Google Books quota exhaustion explicitly", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ error: { code: 429, status: "RESOURCE_EXHAUSTED" } }, 429));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const result = await new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" });
|
||||
await expect(new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" })).rejects.toMatchObject({
|
||||
code: "quota",
|
||||
status: 429
|
||||
} satisfies Partial<GoogleBooksProviderError>);
|
||||
|
||||
expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toContain("q=isbn%3A9782070612376");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("classifies provider DNS failures explicitly", async () => {
|
||||
const error = new TypeError("fetch failed") as Error & { cause?: { code: string } };
|
||||
error.cause = { code: "EAI_AGAIN" };
|
||||
vi.stubGlobal("fetch", vi.fn(async () => Promise.reject(error)));
|
||||
|
||||
await expect(new OpenLibraryProvider().searchByMetadata({ title: "Daredevil", author: null }, config)).rejects.toMatchObject({
|
||||
code: "dns",
|
||||
message: "EAI_AGAIN"
|
||||
} satisfies Partial<MetadataProviderRequestError>);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Demon.Slayer.School.Days.T01.FRENCH.CBZ.eBook-ebdz", "Demon Slayer School Days 1"],
|
||||
["Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", "Dragon Ball SD 1"],
|
||||
["Eyeshield.21.T01.FRENCH.CBZ.eBook-ebdz", "Eyeshield 21 1"],
|
||||
["Solo Leveling T03", "Solo Leveling 3"]
|
||||
])("cleans noisy Google Books title queries for %s", async (title, expectedCleanTitle) => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ totalItems: 0, items: [] }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await new GoogleBooksProvider().searchByMetadata({ title, author: null }, { ...config, provider: "googlebooks" });
|
||||
const queries = fetchMock.mock.calls.map((call) => new URL(String((call as unknown[])[0])).searchParams.get("q") ?? "");
|
||||
|
||||
expect(queries[0]).toBe(`intitle:"${expectedCleanTitle}"`);
|
||||
expect(queries.join(" ")).not.toMatch(/\b(FRENCH|CBZ|eBook|ebdz|Paprika)\b/i);
|
||||
});
|
||||
|
||||
it("sorts Google Books results by matching manga volume instead of taking the first item", async () => {
|
||||
const fetchMock = vi.fn(async () =>
|
||||
jsonResponse({
|
||||
totalItems: 2,
|
||||
items: [
|
||||
{ id: "volume-2", volumeInfo: { title: "Solo Leveling, Vol. 2", authors: ["Chugong"], publishedDate: "2021" } },
|
||||
{ id: "volume-3", volumeInfo: { title: "Solo Leveling, Vol. 3", authors: ["Chugong"], publishedDate: "2021" } }
|
||||
]
|
||||
})
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const results = await new GoogleBooksProvider().searchByMetadata(
|
||||
{ title: "Solo Leveling T03", author: null },
|
||||
{ ...config, provider: "googlebooks" }
|
||||
);
|
||||
|
||||
expect(results[0]).toMatchObject({ title: "Solo Leveling, Vol. 3", sourceId: "volume-3" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Solo Leveling T03", "Solo Leveling"],
|
||||
["Solo Leveling 003", "Solo Leveling"],
|
||||
["Eyeshield.21.T01.FRENCH.CBZ.eBook-ebdz", "Eyeshield 21"],
|
||||
["Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", "Dragon Ball SD"]
|
||||
])("queries MangaDex with the cleaned series title for %s", async (title, expectedQuery) => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ data: [] }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await new MangaDexProvider().searchByMetadata({ title, author: null }, { ...config, provider: "mangadex" });
|
||||
const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0]));
|
||||
|
||||
expect(firstUrl.searchParams.get("title")).toBe(expectedQuery);
|
||||
});
|
||||
|
||||
it("queries MangaDex aliases and maps cover_art to a cover URL", async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(jsonResponse({ data: [] }))
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: "manga-1",
|
||||
attributes: {
|
||||
title: { en: "Demon Slayer: Kimetsu Academy" },
|
||||
description: { en: "School spin-off." },
|
||||
year: 2021,
|
||||
originalLanguage: "ja"
|
||||
},
|
||||
relationships: [
|
||||
{ type: "cover_art", attributes: { fileName: "cover.jpg" } },
|
||||
{ type: "author", attributes: { name: "Natsuki Hokami" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
)
|
||||
.mockResolvedValue(jsonResponse({ data: [] }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const results = await new MangaDexProvider().searchByMetadata(
|
||||
{ title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ.eBook-ebdz", author: null },
|
||||
{ ...config, provider: "mangadex" }
|
||||
);
|
||||
const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0]));
|
||||
const secondUrl = new URL(String((fetchMock.mock.calls[1] as unknown[])[0]));
|
||||
|
||||
expect(firstUrl.searchParams.get("title")).toBe("Demon Slayer School Days");
|
||||
expect(secondUrl.searchParams.get("title")).toBe("Demon Slayer Kimetsu Academy");
|
||||
expect(results[0]).toMatchObject({
|
||||
title: "Demon Slayer: Kimetsu Academy",
|
||||
scoreTitle: "Demon Slayer Kimetsu Academy",
|
||||
author: "Natsuki Hokami",
|
||||
publishedDate: "2021",
|
||||
coverUrl: "https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg"
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Dragon Ball SD ahead of Dragon Ball for MangaDex matches", async () => {
|
||||
const fetchMock = vi.fn(async () =>
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: "dragon-ball",
|
||||
attributes: { title: { en: "Dragon Ball" }, description: { en: "Original series." }, year: 1984, originalLanguage: "ja" },
|
||||
relationships: []
|
||||
},
|
||||
{
|
||||
id: "dragon-ball-sd",
|
||||
attributes: { title: { en: "Dragon Ball SD" }, description: { en: "SD spin-off." }, year: 2010, originalLanguage: "ja" },
|
||||
relationships: []
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const results = await new MangaDexProvider().searchByMetadata(
|
||||
{ title: "Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", author: null },
|
||||
{ ...config, provider: "mangadex" }
|
||||
);
|
||||
|
||||
expect(results[0]).toMatchObject({ title: "Dragon Ball SD", sourceId: "dragon-ball-sd" });
|
||||
});
|
||||
|
||||
it("reports MangaDex rate limits explicitly", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ errors: [{ detail: "Too many requests" }] }, 429)));
|
||||
|
||||
await expect(
|
||||
new MangaDexProvider().searchByMetadata({ title: "Solo Leveling T03", author: null }, { ...config, provider: "mangadex" })
|
||||
).rejects.toMatchObject({ code: "rate-limit", status: 429 });
|
||||
});
|
||||
|
||||
it("requires a Comic Vine API key before querying", async () => {
|
||||
await expect(
|
||||
new ComicVineProvider().searchByMetadata({ title: "Wolverine Origin", author: null }, { ...config, provider: "comicvine", apiKey: null })
|
||||
).rejects.toMatchObject({ code: "missing-key" });
|
||||
});
|
||||
|
||||
it("queries Comic Vine volumes/issues and cleans HTML descriptions", async () => {
|
||||
const fetchMock = vi.fn(async () =>
|
||||
jsonResponse({
|
||||
status_code: 1,
|
||||
results: [
|
||||
{
|
||||
id: 123,
|
||||
name: "Wolverine: The Origin",
|
||||
description: "<p>Origin story & family secrets.</p>",
|
||||
start_year: "2001",
|
||||
image: { super_url: "https://comicvine.gamespot.com/a/uploads/scale_large/origin.jpg" },
|
||||
publisher: { name: "Marvel" }
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const results = await new ComicVineProvider().searchByMetadata(
|
||||
{ title: "Comics.Fr.Wolverine.Origin.by.AleK.(emuleCenter.net)", author: null },
|
||||
{ ...config, provider: "comicvine", apiKey: "cv-key" }
|
||||
);
|
||||
const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0]));
|
||||
|
||||
expect(firstUrl.searchParams.get("resources")).toBe("volume");
|
||||
expect(firstUrl.searchParams.get("query")).toBe("Comics Fr Wolverine Origin by AleK");
|
||||
expect(results[0]).toMatchObject({
|
||||
title: "Wolverine: The Origin",
|
||||
description: "Origin story & family secrets.",
|
||||
publishedDate: "2001",
|
||||
publisher: "Marvel",
|
||||
coverUrl: "https://comicvine.gamespot.com/a/uploads/scale_large/origin.jpg"
|
||||
});
|
||||
});
|
||||
|
||||
it("reports Comic Vine invalid keys and rate limits explicitly", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ status_code: 101, error: "Invalid API Key" })));
|
||||
await expect(
|
||||
new ComicVineProvider().searchByMetadata({ title: "Daredevil", author: null }, { ...config, provider: "comicvine", apiKey: "bad" })
|
||||
).rejects.toMatchObject({ code: "invalid-key" });
|
||||
|
||||
vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ error: "Rate limited" }, 429)));
|
||||
await expect(
|
||||
new ComicVineProvider().searchByMetadata({ title: "Daredevil", author: null }, { ...config, provider: "comicvine", apiKey: "ok" })
|
||||
).rejects.toMatchObject({ code: "rate-limit", status: 429 });
|
||||
});
|
||||
|
||||
it("queries OpenLibrary by local metadata when ISBN is missing", async () => {
|
||||
|
||||
@ -1,14 +1,16 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { DatabaseModule } from "../database/database.module.js";
|
||||
import { BnfProvider } from "./adapters/bnf.provider.js";
|
||||
import { ComicVineProvider } from "./adapters/comic-vine.provider.js";
|
||||
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
|
||||
import { LocalMetadataProvider } from "./adapters/local.provider.js";
|
||||
import { MangaDexProvider } from "./adapters/mangadex.provider.js";
|
||||
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
|
||||
import { MetadataService } from "./metadata.service.js";
|
||||
|
||||
@Module({
|
||||
imports: [DatabaseModule],
|
||||
providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider],
|
||||
providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider, MangaDexProvider, ComicVineProvider],
|
||||
exports: [MetadataService]
|
||||
})
|
||||
export class MetadataModule {}
|
||||
|
||||
@ -1,9 +1,12 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import AdmZip from "adm-zip";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DatabaseService } from "../database/database.service.js";
|
||||
import { books, libraries } from "../database/schema.js";
|
||||
import { BookMetadata } from "../scanner/metadata.js";
|
||||
import { MetadataProviderRequestError } from "./adapters/provider-fetch.js";
|
||||
import { MetadataService } from "./metadata.service.js";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "./metadata.types.js";
|
||||
|
||||
@ -52,7 +55,9 @@ describe("MetadataService", () => {
|
||||
localProvider as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Harry Potter et le Prince de Sang Mele",
|
||||
@ -73,7 +78,7 @@ describe("MetadataService", () => {
|
||||
expect(openLibraryLookup).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup.mock.calls[0]?.[0].identifiers.isbn13).toBe("9782070612383");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et le Prince de sang-mêlé",
|
||||
title: "Harry Potter et le Prince de Sang Mele",
|
||||
isbn: "9782070612383",
|
||||
isbn13: "9782070612383",
|
||||
description: "Harry Potter découvre l'héritage du Prince de Sang-Mêlé."
|
||||
@ -82,6 +87,64 @@ describe("MetadataService", () => {
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("continues enrichment after a provider DNS failure and logs the failure class", async () => {
|
||||
const database = createDatabase();
|
||||
database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'googlebooks'").run();
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => {
|
||||
throw new MetadataProviderRequestError("openlibrary", "dns", "EAI_AGAIN");
|
||||
}
|
||||
};
|
||||
const googleProvider: MetadataProvider = {
|
||||
id: "googlebooks",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "Roy Thomas",
|
||||
description: "Daredevil keeps moving even when another provider is unreachable.",
|
||||
publishedDate: "2019"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
googleProvider as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Daredevil.cbz",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(warn).toHaveBeenCalledWith('[metadata] Provider openlibrary failed for "Daredevil": dns: EAI_AGAIN');
|
||||
expect(result).toMatchObject({
|
||||
title: "Daredevil",
|
||||
author: "Roy Thomas",
|
||||
description: "Daredevil keeps moving even when another provider is unreachable."
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("looks up details from a title search hit even when the hit has no ISBN", async () => {
|
||||
const database = createDatabase();
|
||||
const localProvider = providerStub("local");
|
||||
@ -112,7 +175,9 @@ describe("MetadataService", () => {
|
||||
localProvider as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Harry Potter et le prince de sang mele",
|
||||
@ -133,7 +198,7 @@ describe("MetadataService", () => {
|
||||
expect(openLibraryLookup).toHaveBeenCalledOnce();
|
||||
expect(openLibraryLookup.mock.calls[0]?.[0].sourceId).toBe("OL24333986M");
|
||||
expect(result).toMatchObject({
|
||||
title: "Harry Potter et le prince de sang-mêlé",
|
||||
title: "Harry Potter et le prince de sang mele",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070612383",
|
||||
isbn13: "9782070612383",
|
||||
@ -143,7 +208,61 @@ describe("MetadataService", () => {
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("replaces an ambiguous title-only identification when a later provider supplies a described record", async () => {
|
||||
it.runIf(canLoadBetterSqlite())("backfills a missing ISBN lookup description from a high-confidence title search", async () => {
|
||||
const database = createDatabase();
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => ({
|
||||
title: "Harry Potter et la coupe de feu",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070624553",
|
||||
publisher: "Gallimard",
|
||||
publishedDate: "2016"
|
||||
}),
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Harry Potter et la coupe de feu",
|
||||
author: "J. K. Rowling",
|
||||
isbn: "9782070619207",
|
||||
description: "Harry est invité à assister à la Coupe du monde de Quidditch."
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Harry Potter et la coupe de feu",
|
||||
author: "J. K. Rowling",
|
||||
description: null,
|
||||
isbn: "9782070624553",
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: "/covers/local.jpg"
|
||||
},
|
||||
"/library/Harry Potter et la coupe de feu.epub",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
isbn: "9782070624553",
|
||||
description: "Harry est invité à assister à la Coupe du monde de Quidditch.",
|
||||
coverPath: "/covers/local.jpg"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("uses provider priority as the tie-breaker for high-confidence title matches", async () => {
|
||||
const database = createDatabase();
|
||||
database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'bnf'").run();
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
@ -179,7 +298,9 @@ describe("MetadataService", () => {
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
bnfProvider as never
|
||||
bnfProvider as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
const localMetadata: BookMetadata = {
|
||||
title: "Daredevil",
|
||||
@ -196,14 +317,583 @@ describe("MetadataService", () => {
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: "Daredevil",
|
||||
author: "scénario, Roy Thomas, Gary Friedrich",
|
||||
isbn: "9782809476255",
|
||||
isbn13: "9782809476255",
|
||||
author: "Rosemary Carter",
|
||||
isbn: "9780373105601",
|
||||
isbn13: "9780373105601",
|
||||
description: "Daredevil affronte l'Homme aux échasses."
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("re-enriches from stored local hints instead of a previously failed remote ISBN", async () => {
|
||||
const database = createDatabase();
|
||||
const now = database.now();
|
||||
const library = database.db
|
||||
.insert(libraries)
|
||||
.values({ name: "Comics", path: "/library", enabled: true, createdAt: now, updatedAt: now })
|
||||
.returning()
|
||||
.get();
|
||||
const localMetadataJson = JSON.stringify({
|
||||
title: "Daredevil",
|
||||
author: null,
|
||||
year: null,
|
||||
isbn: null,
|
||||
fileTitle: "Daredevil",
|
||||
raw: { title: "Daredevil", author: null, publishedDate: null, fileName: "Daredevil" }
|
||||
});
|
||||
const book = database.db
|
||||
.insert(books)
|
||||
.values({
|
||||
libraryId: library.id,
|
||||
title: "Daredevil",
|
||||
author: "Rosemary Carter",
|
||||
description: null,
|
||||
isbn: "9780373105601",
|
||||
isbn13: "9780373105601",
|
||||
identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }),
|
||||
localMetadataJson,
|
||||
language: null,
|
||||
publisher: "Harlequin Books",
|
||||
publishedDate: "1982",
|
||||
format: "cbz",
|
||||
filePath: "/library/Daredevil.cbz",
|
||||
coverPath: "/covers/daredevil.jpg",
|
||||
scanStatus: "succeeded",
|
||||
enrichmentStatus: "failed",
|
||||
fileSize: 42,
|
||||
fileMtime: now,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
const openLibraryLookup = vi.fn<(_: MetadataLookup, __: MetadataProviderConfig) => Promise<MetadataMatch | null>>(async () => null);
|
||||
const bnfSearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise<MetadataMatch[]>>(async () => [
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "scénario, Roy Thomas, Gary Friedrich",
|
||||
isbn: "9782809476255",
|
||||
description: "Daredevil affronte l'Homme aux échasses.",
|
||||
publisher: "Panini comics",
|
||||
publishedDate: "2019"
|
||||
}
|
||||
]);
|
||||
database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'bnf'").run();
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
{ id: "openlibrary", lookup: openLibraryLookup, searchByMetadata: async () => [] } as never,
|
||||
providerStub("googlebooks") as never,
|
||||
{ id: "bnf", lookup: async () => null, searchByMetadata: bnfSearch } as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichBook(book.id);
|
||||
|
||||
expect(openLibraryLookup).not.toHaveBeenCalled();
|
||||
expect(bnfSearch.mock.calls[0]?.[0].title).toBe("Daredevil");
|
||||
expect(result).toMatchObject({
|
||||
author: "Rosemary Carter",
|
||||
isbn: "9780373105601",
|
||||
isbn13: "9780373105601",
|
||||
publisher: "Harlequin Books",
|
||||
coverPath: "/covers/daredevil.jpg"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("stores provider covers as local bytes and exposes field provenance with metadata status", async () => {
|
||||
const database = createDatabase();
|
||||
const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "image/jpeg" }),
|
||||
arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer
|
||||
} as Response);
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: "Roy Thomas",
|
||||
description: "Daredevil affronte une nouvelle menace.",
|
||||
isbn: "9782809476255",
|
||||
coverUrl: "https://covers.openlibrary.org/b/id/123-L.jpg"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Daredevil",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Daredevil.cbz",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith("https://covers.openlibrary.org/b/id/123-L.jpg", expect.any(Object));
|
||||
expect(result.coverPath).toMatch(/storage\/covers\/.+\.jpg$/);
|
||||
expect(result.metadataStatus).toBe("enriched");
|
||||
expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({
|
||||
title: "local",
|
||||
author: "openlibrary",
|
||||
description: "openlibrary",
|
||||
coverPath: "openlibrary"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("retrofits a local cover for an existing book during metadata enrichment", async () => {
|
||||
const database = createDatabase();
|
||||
mkdirSync(database.config.storageDir, { recursive: true });
|
||||
const filePath = join(database.config.storageDir, "Demon.Slayer.School.Days.T01.FRENCH.CBZ");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("001.jpg", Buffer.from([0xff, 0xd8, 0xff, 0xd9]));
|
||||
zip.writeZip(filePath);
|
||||
const now = database.now();
|
||||
const library = database.db
|
||||
.insert(libraries)
|
||||
.values({ name: "Comics", path: database.config.storageDir, enabled: true, createdAt: now, updatedAt: now })
|
||||
.returning()
|
||||
.get();
|
||||
const book = database.db
|
||||
.insert(books)
|
||||
.values({
|
||||
libraryId: library.id,
|
||||
title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
isbn13: null,
|
||||
identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }),
|
||||
localMetadataJson: JSON.stringify({
|
||||
title: "Demon Slayer School Days T01 FRENCH",
|
||||
author: null,
|
||||
year: null,
|
||||
isbn: null,
|
||||
fileTitle: "Demon Slayer School Days T01 FRENCH",
|
||||
raw: { title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", author: null, publishedDate: null, fileName: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz" }
|
||||
}),
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
format: "cbz",
|
||||
filePath,
|
||||
coverPath: null,
|
||||
metadataStatus: "none",
|
||||
metadataProvenanceJson: JSON.stringify({ title: "local" }),
|
||||
scanStatus: "succeeded",
|
||||
enrichmentStatus: "succeeded",
|
||||
fileSize: 42,
|
||||
fileMtime: now,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
providerStub("openlibrary") as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichBook(book.id);
|
||||
|
||||
expect(result.coverPath).toMatch(/covers\/[a-f0-9]+\.jpg$/);
|
||||
expect(result.coverPath && existsSync(result.coverPath)).toBe(true);
|
||||
expect(result.metadataStatus).toBe("partial");
|
||||
expect(JSON.parse(result.metadataProvenanceJson ?? "{}")).toMatchObject({ coverPath: "local" });
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("drops sentinel publication dates from provider matches for real affected titles", async () => {
|
||||
const database = createDatabase();
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Harry Potter et les reliques de la mort",
|
||||
author: "J. K. Rowling",
|
||||
publishedDate: "0101-01-01T00:00:00+00:00",
|
||||
description: "Septième année."
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Harry Potter et les reliques de la mort",
|
||||
author: "J. K. Rowling",
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Harry Potter et les reliques de la mort.epub",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(result.publishedDate).toBeNull();
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("does not overwrite an existing valid date with a provider sentinel", async () => {
|
||||
const database = createDatabase();
|
||||
const now = database.now();
|
||||
const library = database.db
|
||||
.insert(libraries)
|
||||
.values({ name: "Novels", path: "/library", enabled: true, createdAt: now, updatedAt: now })
|
||||
.returning()
|
||||
.get();
|
||||
const book = database.db
|
||||
.insert(books)
|
||||
.values({
|
||||
libraryId: library.id,
|
||||
title: "Lord of the Mysteries",
|
||||
author: "Cuttlefish That Loves Diving",
|
||||
description: null,
|
||||
isbn: null,
|
||||
isbn13: null,
|
||||
identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }),
|
||||
localMetadataJson: JSON.stringify({
|
||||
title: "Lord of the Mysteries",
|
||||
author: "Cuttlefish That Loves Diving",
|
||||
year: null,
|
||||
isbn: null,
|
||||
fileTitle: "Lord of the Mysteries",
|
||||
raw: { title: "Lord of the Mysteries", author: "Cuttlefish That Loves Diving", publishedDate: null, fileName: "Lord of the Mysteries" }
|
||||
}),
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: "2018",
|
||||
format: "epub",
|
||||
filePath: "/library/Lord of the Mysteries.epub",
|
||||
coverPath: null,
|
||||
metadataStatus: "partial",
|
||||
metadataProvenanceJson: JSON.stringify({ publishedDate: "existing" }),
|
||||
scanStatus: "succeeded",
|
||||
enrichmentStatus: "succeeded",
|
||||
fileSize: 42,
|
||||
fileMtime: now,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
const openLibraryProvider: MetadataProvider = {
|
||||
id: "openlibrary",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Lord of the Mysteries",
|
||||
author: "Cuttlefish That Loves Diving",
|
||||
publishedDate: "0101-01-01T00:00:00+00:00",
|
||||
description: "A mysterious sequence begins."
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
openLibraryProvider as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
providerStub("mangadex") as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichBook(book.id);
|
||||
|
||||
expect(result.publishedDate).toBe("2018");
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("records MangaDex provenance and stores its cover locally", async () => {
|
||||
const database = createDatabase();
|
||||
const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({
|
||||
ok: true,
|
||||
headers: new Headers({ "content-type": "image/jpeg" }),
|
||||
arrayBuffer: async () => new Uint8Array([9, 8, 7]).buffer
|
||||
} as Response);
|
||||
const mangaDexProvider: MetadataProvider = {
|
||||
id: "mangadex",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Solo Leveling",
|
||||
author: "Chugong",
|
||||
description: "A hunter levels up alone.",
|
||||
publishedDate: "2018",
|
||||
coverUrl: "https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
providerStub("openlibrary") as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
mangaDexProvider as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Solo Leveling T03",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Solo Leveling T03.cbz",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith("https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg", expect.any(Object));
|
||||
expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({
|
||||
author: "mangadex",
|
||||
description: "mangadex",
|
||||
publishedDate: "mangadex",
|
||||
coverPath: "mangadex"
|
||||
});
|
||||
expect(result.coverPath).toMatch(/storage\/covers\/.+\.jpg$/);
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("scores MangaDex matches against the cleaned series title instead of the noisy archive title", async () => {
|
||||
const database = createDatabase();
|
||||
const mangaDexProvider: MetadataProvider = {
|
||||
id: "mangadex",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Dragon Ball SD",
|
||||
author: "Naho Ooishi",
|
||||
description: "A super-deformed Dragon Ball spin-off.",
|
||||
publishedDate: "2010",
|
||||
sourceId: "dragon-ball-sd"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
providerStub("openlibrary") as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
mangaDexProvider as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+.cbz",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: "Dragon Ball SD",
|
||||
author: "Naho Ooishi",
|
||||
description: "A super-deformed Dragon Ball spin-off.",
|
||||
publishedDate: "2010"
|
||||
});
|
||||
expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({
|
||||
title: "local",
|
||||
author: "mangadex",
|
||||
description: "mangadex"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("accepts MangaDex alias matches through the provider score title", async () => {
|
||||
const database = createDatabase();
|
||||
const mangaDexProvider: MetadataProvider = {
|
||||
id: "mangadex",
|
||||
lookup: async () => null,
|
||||
searchByMetadata: async () => [
|
||||
{
|
||||
title: "Demon Slayer: Kimetsu Academy",
|
||||
scoreTitle: "Demon Slayer Kimetsu Academy",
|
||||
author: "Natsuki Hokami",
|
||||
description: "School spin-off.",
|
||||
publishedDate: "2021",
|
||||
sourceId: "kimetsu-academy"
|
||||
}
|
||||
]
|
||||
};
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
providerStub("openlibrary") as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
mangaDexProvider as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichMetadata(
|
||||
{
|
||||
title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
coverPath: null
|
||||
},
|
||||
"/library/Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz.cbz",
|
||||
{ remote: true }
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: "Demon Slayer School Days",
|
||||
author: "Natsuki Hokami",
|
||||
description: "School spin-off.",
|
||||
publishedDate: "2021"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
|
||||
it.runIf(canLoadBetterSqlite())("re-enriches existing manga with the cleaned series title on the live book path", async () => {
|
||||
const database = createDatabase();
|
||||
mkdirSync(database.config.storageDir, { recursive: true });
|
||||
const filePath = join(database.config.storageDir, "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz.cbz");
|
||||
const zip = new AdmZip();
|
||||
zip.addFile("001.jpg", Buffer.from([0xff, 0xd8, 0xff, 0xd9]));
|
||||
zip.writeZip(filePath);
|
||||
const now = database.now();
|
||||
const library = database.db
|
||||
.insert(libraries)
|
||||
.values({ name: "Manga", path: database.config.storageDir, enabled: true, createdAt: now, updatedAt: now })
|
||||
.returning()
|
||||
.get();
|
||||
const book = database.db
|
||||
.insert(books)
|
||||
.values({
|
||||
libraryId: library.id,
|
||||
title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz",
|
||||
author: null,
|
||||
description: null,
|
||||
isbn: null,
|
||||
isbn13: null,
|
||||
identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }),
|
||||
localMetadataJson: JSON.stringify({
|
||||
title: "Demon Slayer School Days T01 FRENCH",
|
||||
author: null,
|
||||
year: null,
|
||||
isbn: null,
|
||||
fileTitle: "Demon Slayer School Days T01 FRENCH",
|
||||
raw: {
|
||||
title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz",
|
||||
author: null,
|
||||
publishedDate: null,
|
||||
fileName: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz"
|
||||
}
|
||||
}),
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
format: "cbz",
|
||||
filePath,
|
||||
coverPath: null,
|
||||
metadataStatus: "none",
|
||||
metadataProvenanceJson: JSON.stringify({ title: "local" }),
|
||||
scanStatus: "succeeded",
|
||||
enrichmentStatus: "succeeded",
|
||||
fileSize: 42,
|
||||
fileMtime: now,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
const mangaDexSearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise<MetadataMatch[]>>(async () => [
|
||||
{
|
||||
title: "Demon Slayer School Days",
|
||||
author: "Natsuki Hokami",
|
||||
description: "School spin-off.",
|
||||
publishedDate: "2021",
|
||||
sourceId: "demon-slayer-school-days"
|
||||
}
|
||||
]);
|
||||
const service = new MetadataService(
|
||||
database,
|
||||
providerStub("local") as never,
|
||||
providerStub("openlibrary") as never,
|
||||
providerStub("googlebooks") as never,
|
||||
providerStub("bnf") as never,
|
||||
{ id: "mangadex", lookup: async () => null, searchByMetadata: mangaDexSearch } as never,
|
||||
providerStub("comicvine") as never
|
||||
);
|
||||
|
||||
const result = await service.enrichBook(book.id);
|
||||
|
||||
expect(mangaDexSearch.mock.calls[0]?.[0].title).toBe("Demon Slayer School Days");
|
||||
expect(result).toMatchObject({
|
||||
title: "Demon Slayer School Days",
|
||||
author: "Natsuki Hokami",
|
||||
description: "School spin-off.",
|
||||
publishedDate: "2021"
|
||||
});
|
||||
|
||||
database.onModuleDestroy();
|
||||
});
|
||||
});
|
||||
|
||||
function createDatabase(): DatabaseService {
|
||||
|
||||
@ -1,21 +1,45 @@
|
||||
import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname, extname, join } from "node:path";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
MetadataSourcesConfigDto,
|
||||
UpdateMetadataSourcesConfigDto
|
||||
} from "@readabook/shared";
|
||||
import { DatabaseService } from "../database/database.service.js";
|
||||
import { automationSettings, books, metadataSourceConfig } from "../database/schema.js";
|
||||
import { BookMetadata } from "../scanner/metadata.js";
|
||||
import { automationSettings, books, metadataSourceConfig, series } from "../database/schema.js";
|
||||
import { BookMetadata, extractMetadata } from "../scanner/metadata.js";
|
||||
import { BnfProvider } from "./adapters/bnf.provider.js";
|
||||
import { ComicVineProvider } from "./adapters/comic-vine.provider.js";
|
||||
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
|
||||
import { LocalMetadataProvider } from "./adapters/local.provider.js";
|
||||
import { MangaDexProvider } from "./adapters/mangadex.provider.js";
|
||||
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
|
||||
import { BookIdentifiers, LocalMetadataHints, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js";
|
||||
import { describeMetadataProviderError, providerFetch } from "./adapters/provider-fetch.js";
|
||||
import {
|
||||
BookIdentifiers,
|
||||
LocalMetadataHints,
|
||||
MetadataField,
|
||||
MetadataMatch,
|
||||
MetadataProvider,
|
||||
MetadataProviderConfig,
|
||||
MetadataProviderId,
|
||||
MetadataProvenance,
|
||||
MetadataSearchQuery,
|
||||
MetadataStatus
|
||||
} from "./metadata.types.js";
|
||||
import { ExtractIdentifiers, toIsbn13 } from "./use-cases/extract-identifiers.js";
|
||||
import { ExtractLocalMetadataHints } from "./use-cases/extract-local-metadata-hints.js";
|
||||
import { extractSeriesVolume } from "./use-cases/extract-series-volume.js";
|
||||
import { normalizePublishedDate } from "./use-cases/normalize-published-date.js";
|
||||
import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js";
|
||||
import { ScoreMetadataMatch } from "./use-cases/score-metadata-match.js";
|
||||
import { ScoredMetadataMatch, ScoreMetadataMatch } from "./use-cases/score-metadata-match.js";
|
||||
|
||||
type ProviderCandidate = ScoredMetadataMatch & {
|
||||
provider: MetadataProviderId;
|
||||
priority: number;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class MetadataService {
|
||||
@ -29,9 +53,11 @@ export class MetadataService {
|
||||
local: LocalMetadataProvider,
|
||||
openLibrary: OpenLibraryProvider,
|
||||
googleBooks: GoogleBooksProvider,
|
||||
bnf: BnfProvider
|
||||
bnf: BnfProvider,
|
||||
mangaDex: MangaDexProvider,
|
||||
comicVine: ComicVineProvider
|
||||
) {
|
||||
this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf]);
|
||||
this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf, mangaDex, comicVine]);
|
||||
}
|
||||
|
||||
getSourcesConfig(): MetadataSourcesConfigDto {
|
||||
@ -75,96 +101,119 @@ export class MetadataService {
|
||||
localMetadata: BookMetadata,
|
||||
filePath: string,
|
||||
options: { remote: boolean }
|
||||
): Promise<BookMetadata & { isbn13: string | null; identifiersJson: string; localMetadataJson: string }> {
|
||||
): Promise<
|
||||
BookMetadata & {
|
||||
isbn13: string | null;
|
||||
identifiersJson: string;
|
||||
localMetadataJson: string;
|
||||
metadataStatus: MetadataStatus;
|
||||
metadataProvenanceJson: string;
|
||||
}
|
||||
> {
|
||||
const identifiers = this.extractIdentifiers.fromMetadataAndFile(localMetadata, filePath);
|
||||
const local = this.extractLocalMetadataHints.fromMetadataAndFile(localMetadata, filePath);
|
||||
const configs = this.getProviderConfigs();
|
||||
const chain = options.remote
|
||||
? this.resolveProviderChain.resolve(configs)
|
||||
: this.resolveProviderChain.resolve(configs).filter((entry) => entry.provider.id === "local");
|
||||
let merged: BookMetadata = { ...localMetadata };
|
||||
const candidates: ProviderCandidate[] = [];
|
||||
const query = this.buildSearchQuery(local, identifiers, filePath);
|
||||
|
||||
for (const { provider, config } of chain) {
|
||||
if (provider.id === "local") continue;
|
||||
try {
|
||||
const hasIsbn = Boolean(identifiers.isbn13 ?? identifiers.isbn10);
|
||||
const match =
|
||||
provider.id === "local" || hasIsbn
|
||||
? await provider.lookup(
|
||||
{
|
||||
title: merged.title,
|
||||
author: merged.author,
|
||||
filePath,
|
||||
sourceId: null,
|
||||
identifiers,
|
||||
local
|
||||
},
|
||||
config
|
||||
)
|
||||
: null;
|
||||
const match = hasIsbn
|
||||
? await provider.lookup(
|
||||
{
|
||||
title: local.title,
|
||||
author: local.author,
|
||||
filePath,
|
||||
sourceId: null,
|
||||
identifiers,
|
||||
local
|
||||
},
|
||||
config
|
||||
)
|
||||
: null;
|
||||
if (match) {
|
||||
merged = mergeMetadata(merged, match);
|
||||
const completedMatch = match.description
|
||||
? match
|
||||
: mergeMetadataMatch(match, await this.searchMissingDescription(provider, config, local, identifiers, filePath));
|
||||
candidates.push(scoreProviderCandidate(this.scoreMetadataMatch, query, completedMatch, provider.id, config.priority));
|
||||
continue;
|
||||
}
|
||||
if (!options.remote || provider.id === "local") continue;
|
||||
const query = {
|
||||
title: local.title,
|
||||
author: local.author,
|
||||
year: local.year,
|
||||
isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn
|
||||
};
|
||||
const best = this.scoreMetadataMatch.best(query, await provider.searchByMetadata(query, config));
|
||||
if (!options.remote) continue;
|
||||
const matches = await provider.searchByMetadata(query, config);
|
||||
const best = this.scoreMetadataMatch.best(query, matches);
|
||||
if (!best && matches.length) {
|
||||
console.info(
|
||||
`[metadata] Provider ${provider.id} returned ${matches.length} result(s) rejected by scoring for "${query.title}"`
|
||||
);
|
||||
}
|
||||
if (best) {
|
||||
if (!isActionableSearchMatch(best.match)) continue;
|
||||
merged = shouldReplaceAmbiguousIdentification(localMetadata, merged, best.match)
|
||||
? mergeMetadata({ ...localMetadata, author: null, isbn: null, description: null, language: null, publisher: null, publishedDate: null }, best.match)
|
||||
: mergeMetadata(merged, best.match);
|
||||
let providerMatch = best.match;
|
||||
|
||||
const detailedMatch = await this.lookupSearchMatchDetails(provider, config, filePath, identifiers, local, merged, best.match);
|
||||
if (detailedMatch) merged = mergeMetadata(merged, detailedMatch);
|
||||
const detailedMatch = await this.lookupSearchMatchDetails(provider, config, filePath, identifiers, local, best.match);
|
||||
if (detailedMatch) providerMatch = mergeMetadataMatch(detailedMatch, best.match);
|
||||
candidates.push(scoreProviderCandidate(this.scoreMetadataMatch, query, providerMatch, provider.id, config.priority));
|
||||
}
|
||||
} catch {
|
||||
// Provider failures must not block local ingestion.
|
||||
} catch (error) {
|
||||
console.warn(`[metadata] Provider ${provider.id} failed for "${query.title}": ${describeMetadataProviderError(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
const materializedCandidates = await this.materializeQualifiedCovers(candidates, filePath);
|
||||
const localProvenance = provenanceFromLocal(localMetadata);
|
||||
const { metadata: merged, provenance: remoteProvenance } = mergeCandidatesWithLocal(
|
||||
materializedCandidates,
|
||||
localMetadata,
|
||||
identifiers,
|
||||
query.title
|
||||
);
|
||||
let provenance: MetadataProvenance = { ...localProvenance, ...remoteProvenance };
|
||||
if (merged.isbn && !provenance.isbn) provenance.isbn = "local";
|
||||
const isbn13 = identifiers.isbn13 ?? (merged.isbn ? toIsbn13(merged.isbn) : null);
|
||||
const metadataStatus = computeMetadataStatus(merged);
|
||||
return {
|
||||
...merged,
|
||||
isbn: merged.isbn ?? isbn13 ?? identifiers.isbn10,
|
||||
isbn13,
|
||||
identifiersJson: JSON.stringify(identifiers),
|
||||
localMetadataJson: JSON.stringify(local)
|
||||
localMetadataJson: JSON.stringify(local),
|
||||
metadataStatus,
|
||||
metadataProvenanceJson: JSON.stringify(provenance)
|
||||
};
|
||||
}
|
||||
|
||||
async enrichBook(bookId: number): Promise<typeof books.$inferSelect> {
|
||||
const book = this.database.db.select().from(books).where(eq(books.id, bookId)).get();
|
||||
if (!book) throw new NotFoundException("Book not found");
|
||||
const metadata: BookMetadata = {
|
||||
title: book.title,
|
||||
author: book.author,
|
||||
description: book.description,
|
||||
isbn: book.isbn,
|
||||
language: book.language,
|
||||
publisher: book.publisher,
|
||||
publishedDate: book.publishedDate,
|
||||
coverPath: book.coverPath
|
||||
};
|
||||
const local = parseStoredLocalMetadata(book.localMetadataJson);
|
||||
const metadata = await this.extractCurrentLocalMetadata(book, local);
|
||||
const enriched = await this.enrichMetadata(metadata, book.filePath, { remote: true });
|
||||
const next = preserveExistingWhenMissing(enriched, book, book.filePath);
|
||||
const seriesInfo = this.resolveSeries(next.title, book.filePath);
|
||||
return this.database.db
|
||||
.update(books)
|
||||
.set({
|
||||
title: enriched.title,
|
||||
author: enriched.author,
|
||||
description: enriched.description,
|
||||
isbn: enriched.isbn,
|
||||
isbn13: enriched.isbn13,
|
||||
seriesId: seriesInfo.seriesId,
|
||||
title: next.title,
|
||||
author: next.author,
|
||||
description: next.description,
|
||||
isbn: next.isbn,
|
||||
isbn13: next.isbn13,
|
||||
identifiersJson: enriched.identifiersJson,
|
||||
localMetadataJson: enriched.localMetadataJson,
|
||||
language: enriched.language,
|
||||
publisher: enriched.publisher,
|
||||
publishedDate: enriched.publishedDate,
|
||||
coverPath: enriched.coverPath,
|
||||
language: next.language,
|
||||
publisher: next.publisher,
|
||||
publishedDate: next.publishedDate,
|
||||
volumeNumber: seriesInfo.volumeNumber,
|
||||
volumeLabel: seriesInfo.volumeLabel,
|
||||
coverPath: next.coverPath,
|
||||
metadataStatus: next.metadataStatus,
|
||||
metadataProvenanceJson: next.metadataProvenanceJson,
|
||||
updatedAt: this.database.now()
|
||||
})
|
||||
.where(eq(books.id, book.id))
|
||||
@ -172,6 +221,57 @@ export class MetadataService {
|
||||
.get();
|
||||
}
|
||||
|
||||
private async extractCurrentLocalMetadata(book: typeof books.$inferSelect, local: LocalMetadataHints | null): Promise<BookMetadata> {
|
||||
const fallback: BookMetadata = {
|
||||
title: local?.title ?? book.title,
|
||||
author: local ? local.author : book.author,
|
||||
description: null,
|
||||
isbn: local ? local.isbn : book.isbn,
|
||||
language: null,
|
||||
publisher: book.publisher,
|
||||
publishedDate: normalizePublishedDate(local ? local.year : book.publishedDate),
|
||||
coverPath: book.coverPath
|
||||
};
|
||||
if (!existsSync(book.filePath)) return fallback;
|
||||
try {
|
||||
const extracted = await extractMetadata(book.filePath, this.database.config.storageDir);
|
||||
return {
|
||||
title: extracted.title || fallback.title,
|
||||
author: extracted.author ?? fallback.author,
|
||||
description: extracted.description ?? fallback.description,
|
||||
isbn: extracted.isbn ?? fallback.isbn,
|
||||
language: extracted.language ?? fallback.language,
|
||||
publisher: extracted.publisher ?? fallback.publisher,
|
||||
publishedDate: normalizePublishedDate(extracted.publishedDate) ?? fallback.publishedDate,
|
||||
coverPath: extracted.coverPath ?? fallback.coverPath
|
||||
};
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } {
|
||||
const parsed = extractSeriesVolume(title, filePath);
|
||||
const now = this.database.now();
|
||||
const row = this.database.db
|
||||
.insert(series)
|
||||
.values({
|
||||
title: parsed.seriesTitle,
|
||||
normalizedTitle: parsed.normalizedSeriesTitle,
|
||||
description: null,
|
||||
publisher: null,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: series.normalizedTitle,
|
||||
set: { title: parsed.seriesTitle, updatedAt: now }
|
||||
})
|
||||
.returning({ id: series.id })
|
||||
.get();
|
||||
return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel };
|
||||
}
|
||||
|
||||
private getProviderConfigs(): MetadataProviderConfig[] {
|
||||
return this.database.db
|
||||
.select()
|
||||
@ -195,7 +295,6 @@ export class MetadataService {
|
||||
filePath: string,
|
||||
identifiers: BookIdentifiers,
|
||||
local: LocalMetadataHints,
|
||||
merged: BookMetadata,
|
||||
match: MetadataMatch
|
||||
): Promise<MetadataMatch | null> {
|
||||
const derivedIdentifiers = {
|
||||
@ -206,13 +305,12 @@ export class MetadataService {
|
||||
};
|
||||
const hasNewIdentifier = derivedIdentifiers.isbn13 !== identifiers.isbn13 || derivedIdentifiers.isbn10 !== identifiers.isbn10;
|
||||
const hasLookupTarget = hasNewIdentifier || Boolean(match.sourceId);
|
||||
const needsDetails = !merged.description && hasLookupTarget;
|
||||
if (!hasLookupTarget && !needsDetails) return null;
|
||||
if (!hasLookupTarget) return null;
|
||||
|
||||
return provider.lookup(
|
||||
{
|
||||
title: merged.title,
|
||||
author: merged.author,
|
||||
title: match.title ?? local.title,
|
||||
author: match.author ?? local.author,
|
||||
filePath,
|
||||
sourceId: match.sourceId,
|
||||
identifiers: derivedIdentifiers,
|
||||
@ -221,26 +319,309 @@ export class MetadataService {
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
private async searchMissingDescription(
|
||||
provider: MetadataProvider,
|
||||
config: MetadataProviderConfig,
|
||||
local: LocalMetadataHints,
|
||||
identifiers: BookIdentifiers,
|
||||
filePath: string
|
||||
): Promise<MetadataMatch | null> {
|
||||
const query = this.buildSearchQuery(local, identifiers, filePath);
|
||||
const matches = await provider.searchByMetadata(query, config);
|
||||
return (
|
||||
matches
|
||||
.map((match) => ({ match, score: this.scoreMetadataMatch.score(query, match) }))
|
||||
.filter((entry) => entry.match.description && entry.score >= 75)
|
||||
.sort((left, right) => right.score - left.score)[0]?.match ?? null
|
||||
);
|
||||
}
|
||||
|
||||
private buildSearchQuery(local: LocalMetadataHints, identifiers: BookIdentifiers, filePath: string): MetadataSearchQuery {
|
||||
return {
|
||||
title: extractSeriesVolume(local.title, filePath).seriesTitle,
|
||||
author: local.author,
|
||||
year: local.year,
|
||||
isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn
|
||||
};
|
||||
}
|
||||
|
||||
private async materializeCover(match: MetadataMatch, filePath: string, provider: MetadataProviderId): Promise<MetadataMatch> {
|
||||
if (match.coverPath || !match.coverUrl) return match;
|
||||
try {
|
||||
const response = await providerFetch("cover", match.coverUrl, { timeoutMs: 5000 });
|
||||
if (!response.ok) return match;
|
||||
const data = Buffer.from(await response.arrayBuffer());
|
||||
if (!data.length) return match;
|
||||
const extension = coverExtension(match.coverUrl, response.headers.get("content-type"));
|
||||
const hash = createHash("sha256").update(`${filePath}:${provider}:${match.coverUrl}`).digest("hex").slice(0, 24);
|
||||
const target = join(this.database.config.storageDir, "covers", `${hash}${extension}`);
|
||||
mkdirSync(dirname(target), { recursive: true });
|
||||
writeFileSync(target, data);
|
||||
return { ...match, coverPath: target };
|
||||
} catch {
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
private async materializeQualifiedCovers(candidates: ProviderCandidate[], filePath: string): Promise<ProviderCandidate[]> {
|
||||
const materialized: ProviderCandidate[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (isQualifiedSourceCover(candidate)) {
|
||||
materialized.push({
|
||||
...candidate,
|
||||
match: await this.materializeCover(candidate.match, filePath, candidate.provider)
|
||||
});
|
||||
} else {
|
||||
materialized.push(candidate);
|
||||
}
|
||||
}
|
||||
return materialized;
|
||||
}
|
||||
}
|
||||
|
||||
function isActionableSearchMatch(match: MetadataMatch): boolean {
|
||||
return Boolean(match.isbn ?? match.sourceId ?? match.description);
|
||||
}
|
||||
|
||||
function shouldReplaceAmbiguousIdentification(local: BookMetadata, current: BookMetadata, next: MetadataMatch): boolean {
|
||||
if (local.author || local.isbn || !current.isbn || !next.isbn || current.isbn === next.isbn) return false;
|
||||
return Boolean(next.description);
|
||||
}
|
||||
|
||||
function mergeMetadata(current: BookMetadata, next: MetadataMatch): BookMetadata {
|
||||
function mergeMetadataMatch(current: MetadataMatch, next: MetadataMatch | null): MetadataMatch {
|
||||
if (!next) return current;
|
||||
return {
|
||||
title: next.title ?? current.title,
|
||||
author: current.author ?? next.author ?? null,
|
||||
description: current.description ?? next.description ?? null,
|
||||
isbn: current.isbn ?? next.isbn ?? null,
|
||||
language: current.language ?? next.language ?? null,
|
||||
publisher: current.publisher ?? next.publisher ?? null,
|
||||
publishedDate: current.publishedDate ?? next.publishedDate ?? null,
|
||||
coverPath: current.coverPath ?? next.coverPath ?? null
|
||||
title: current.title ?? next.title,
|
||||
author: current.author ?? next.author,
|
||||
description: current.description ?? next.description,
|
||||
isbn: current.isbn ?? next.isbn,
|
||||
language: current.language ?? next.language,
|
||||
publisher: current.publisher ?? next.publisher,
|
||||
publishedDate: normalizePublishedDate(current.publishedDate) ?? normalizePublishedDate(next.publishedDate),
|
||||
coverPath: current.coverPath ?? next.coverPath,
|
||||
coverUrl: current.coverUrl ?? next.coverUrl,
|
||||
sourceId: current.sourceId ?? next.sourceId,
|
||||
identifiers: current.identifiers ?? next.identifiers
|
||||
};
|
||||
}
|
||||
|
||||
const metadataFields: MetadataField[] = ["title", "author", "description", "isbn", "language", "publisher", "publishedDate", "coverPath"];
|
||||
const fillableMetadataFields: MetadataField[] = ["author", "description", "isbn", "language", "publisher", "publishedDate"];
|
||||
|
||||
function scoreProviderCandidate(
|
||||
scorer: ScoreMetadataMatch,
|
||||
query: MetadataSearchQuery,
|
||||
match: MetadataMatch,
|
||||
provider: MetadataProviderId,
|
||||
priority: number
|
||||
): ProviderCandidate {
|
||||
return { ...scorer.details(query, match), provider, priority };
|
||||
}
|
||||
|
||||
function mergeCandidatesWithLocal(
|
||||
candidates: ProviderCandidate[],
|
||||
local: BookMetadata,
|
||||
identifiers: BookIdentifiers,
|
||||
title: string
|
||||
): { metadata: BookMetadata; provenance: MetadataProvenance } {
|
||||
const sorted = [...candidates].sort(compareProviderCandidates);
|
||||
const retained = sorted[0] ?? null;
|
||||
const completionOrder = [...(retained ? [retained] : []), ...sorted.filter((candidate) => candidate !== retained)];
|
||||
const metadata: BookMetadata = {
|
||||
title,
|
||||
author: local.author ?? null,
|
||||
description: local.description ?? null,
|
||||
isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn ?? null,
|
||||
language: local.language ?? null,
|
||||
publisher: local.publisher ?? null,
|
||||
publishedDate: normalizePublishedDate(local.publishedDate),
|
||||
coverPath: local.coverPath ?? null
|
||||
};
|
||||
const provenance: MetadataProvenance = {};
|
||||
for (const field of fillableMetadataFields) {
|
||||
if (hasMetadataValue(metadata[field])) continue;
|
||||
const source = completionOrder.find((candidate) => hasMetadataValue(normalizeCandidateField(candidate.match, field)));
|
||||
if (!source) continue;
|
||||
metadata[field] = normalizeCandidateField(source.match, field) as never;
|
||||
provenance[field] = source.provider;
|
||||
provenance[`${field}Score` as MetadataField] = String(source.score) as never;
|
||||
}
|
||||
const coverSource = completionOrder.find((candidate) => isQualifiedSourceCover(candidate) && hasMetadataValue(candidate.match.coverPath));
|
||||
if (coverSource && shouldUseSourceCover(metadata.coverPath)) {
|
||||
metadata.coverPath = coverSource.match.coverPath ?? null;
|
||||
provenance.coverPath = coverSource.provider;
|
||||
provenance.coverPathScore = String(coverSource.score) as never;
|
||||
}
|
||||
return { metadata, provenance };
|
||||
}
|
||||
|
||||
function compareProviderCandidates(left: ProviderCandidate, right: ProviderCandidate): number {
|
||||
if (left.isbnMatch !== right.isbnMatch) return left.isbnMatch ? -1 : 1;
|
||||
const leftHighConfidence = isHighConfidenceSelection(left);
|
||||
const rightHighConfidence = isHighConfidenceSelection(right);
|
||||
if (leftHighConfidence && rightHighConfidence) return left.priority - right.priority;
|
||||
if (leftHighConfidence !== rightHighConfidence) return leftHighConfidence ? -1 : 1;
|
||||
if (left.score !== right.score) return right.score - left.score;
|
||||
return left.priority - right.priority;
|
||||
}
|
||||
|
||||
function isHighConfidenceSelection(candidate: ProviderCandidate): boolean {
|
||||
return candidate.titleScore > 90 && (candidate.authorScore == null || candidate.authorScore >= 15);
|
||||
}
|
||||
|
||||
function isQualifiedSourceCover(candidate: ProviderCandidate): boolean {
|
||||
return candidate.score >= 80 && candidate.titleScore >= 85 && Boolean(candidate.match.coverPath ?? candidate.match.coverUrl);
|
||||
}
|
||||
|
||||
function shouldUseSourceCover(currentCoverPath: string | null): boolean {
|
||||
return !hasMetadataValue(currentCoverPath) || isLocalCoverPath(currentCoverPath);
|
||||
}
|
||||
|
||||
function isLocalCoverPath(value: string): boolean {
|
||||
return /[/\\]covers[/\\][a-f0-9]{24}\.[a-z0-9]+$/i.test(value);
|
||||
}
|
||||
|
||||
function normalizeCandidateField(match: MetadataMatch, field: MetadataField): string | null {
|
||||
if (field === "publishedDate") return normalizePublishedDate(match.publishedDate);
|
||||
return match[field] ?? null;
|
||||
}
|
||||
|
||||
function hasMetadataValue(value: string | null | undefined): value is string {
|
||||
return Boolean(value && value.trim());
|
||||
}
|
||||
|
||||
function provenanceFromLocal(local: BookMetadata): MetadataProvenance {
|
||||
const provenance: MetadataProvenance = {};
|
||||
for (const field of metadataFields) {
|
||||
if (local[field] != null && local[field] !== "") provenance[field] = "local";
|
||||
}
|
||||
return provenance;
|
||||
}
|
||||
|
||||
function computeMetadataStatus(metadata: Pick<BookMetadata, "author" | "description" | "isbn" | "language" | "publisher" | "publishedDate" | "coverPath">): MetadataStatus {
|
||||
const hasCover = Boolean(metadata.coverPath);
|
||||
const filled = [metadata.author, metadata.description, metadata.isbn, metadata.language, metadata.publisher, metadata.publishedDate].filter(Boolean).length;
|
||||
if (hasCover && filled >= 2) return "enriched";
|
||||
if (hasCover || filled > 0) return "partial";
|
||||
return "none";
|
||||
}
|
||||
|
||||
function mergeExistingProvenance(
|
||||
enriched: BookMetadata & { metadataProvenanceJson: string },
|
||||
existing: typeof books.$inferSelect,
|
||||
finalValues: BookMetadata & { isbn13: string | null }
|
||||
): MetadataProvenance {
|
||||
const next = parseProvenance(enriched.metadataProvenanceJson);
|
||||
const previous = parseProvenance(existing.metadataProvenanceJson);
|
||||
const provenance: MetadataProvenance = { ...previous };
|
||||
for (const field of metadataFields) {
|
||||
if (field === "title") {
|
||||
if (finalValues.title && !provenance.title) {
|
||||
provenance.title = finalValues.title === enriched.title && finalValues.title !== existing.title ? (next.title ?? "local") : (previous.title ?? "existing");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (field === "publishedDate") {
|
||||
const finalDate = normalizePublishedDate(finalValues.publishedDate);
|
||||
if (finalDate && finalDate === normalizePublishedDate(enriched.publishedDate) && finalDate !== normalizePublishedDate(existing.publishedDate)) {
|
||||
provenance[field] = next[field] ?? provenance[field];
|
||||
copyScoreProvenance(next, provenance, field);
|
||||
} else if (finalDate && !provenance[field]) {
|
||||
provenance[field] = previous[field] ?? "existing";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (field === "coverPath" && finalValues.coverPath && finalValues.coverPath === enriched.coverPath && finalValues.coverPath !== existing.coverPath) {
|
||||
provenance.coverPath = next.coverPath ?? provenance.coverPath;
|
||||
copyScoreProvenance(next, provenance, field);
|
||||
continue;
|
||||
}
|
||||
if (finalValues[field] && finalValues[field] === enriched[field] && finalValues[field] !== existing[field]) {
|
||||
provenance[field] = next[field] ?? provenance[field];
|
||||
copyScoreProvenance(next, provenance, field);
|
||||
continue;
|
||||
}
|
||||
if (finalValues[field] != null && existing[field] != null && !provenance[field]) {
|
||||
provenance[field] = previous[field] ?? "existing";
|
||||
}
|
||||
}
|
||||
return provenance;
|
||||
}
|
||||
|
||||
function copyScoreProvenance(source: MetadataProvenance, target: MetadataProvenance, field: MetadataField): void {
|
||||
const scoreKey = `${field}Score`;
|
||||
if (source[scoreKey]) target[scoreKey] = source[scoreKey];
|
||||
}
|
||||
|
||||
function parseProvenance(value: string | null): MetadataProvenance {
|
||||
if (!value) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(value) as MetadataProvenance;
|
||||
return parsed && typeof parsed === "object" ? parsed : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function coverExtension(url: string, contentType: string | null): string {
|
||||
if (contentType?.includes("png")) return ".png";
|
||||
if (contentType?.includes("webp")) return ".webp";
|
||||
if (contentType?.includes("gif")) return ".gif";
|
||||
const fromUrl = extname(new URL(url).pathname).toLowerCase();
|
||||
return fromUrl === ".png" || fromUrl === ".webp" || fromUrl === ".gif" || fromUrl === ".jpg" || fromUrl === ".jpeg" ? fromUrl : ".jpg";
|
||||
}
|
||||
|
||||
function parseStoredLocalMetadata(value: string | null): LocalMetadataHints | null {
|
||||
if (!value) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(value) as Partial<LocalMetadataHints>;
|
||||
return typeof parsed.title === "string" ? (parsed as LocalMetadataHints) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function preserveExistingWhenMissing(
|
||||
enriched: BookMetadata & {
|
||||
isbn13: string | null;
|
||||
identifiersJson: string;
|
||||
localMetadataJson: string;
|
||||
metadataStatus: MetadataStatus;
|
||||
metadataProvenanceJson: string;
|
||||
},
|
||||
existing: typeof books.$inferSelect,
|
||||
filePath: string
|
||||
): BookMetadata & { isbn13: string | null; metadataStatus: MetadataStatus; metadataProvenanceJson: string } {
|
||||
const previousProvenance = parseProvenance(existing.metadataProvenanceJson);
|
||||
const next = {
|
||||
title: chooseTitle(existing.title, enriched.title, filePath),
|
||||
author: existing.author ?? enriched.author,
|
||||
description: existing.description ?? enriched.description,
|
||||
isbn: existing.isbn ?? enriched.isbn,
|
||||
isbn13: existing.isbn13 ?? enriched.isbn13,
|
||||
language: existing.language ?? enriched.language,
|
||||
publisher: existing.publisher ?? enriched.publisher,
|
||||
publishedDate: normalizePublishedDate(existing.publishedDate) ?? normalizePublishedDate(enriched.publishedDate),
|
||||
coverPath: chooseCoverPath(existing.coverPath, enriched.coverPath, previousProvenance)
|
||||
};
|
||||
const provenance = mergeExistingProvenance(enriched, existing, next);
|
||||
return {
|
||||
...next,
|
||||
metadataStatus: computeMetadataStatus(next),
|
||||
metadataProvenanceJson: JSON.stringify(provenance)
|
||||
};
|
||||
}
|
||||
|
||||
function chooseCoverPath(existingCoverPath: string | null, enrichedCoverPath: string | null, previousProvenance: MetadataProvenance): string | null {
|
||||
if (!existingCoverPath) return enrichedCoverPath;
|
||||
if (!enrichedCoverPath || enrichedCoverPath === existingCoverPath) return existingCoverPath;
|
||||
return canReplaceExistingCover(existingCoverPath, previousProvenance) ? enrichedCoverPath : existingCoverPath;
|
||||
}
|
||||
|
||||
function chooseTitle(existingTitle: string, enrichedTitle: string, filePath: string): string {
|
||||
if (!enrichedTitle) return existingTitle;
|
||||
if (!existingTitle) return enrichedTitle;
|
||||
const parsedExisting = extractSeriesVolume(existingTitle, filePath).seriesTitle;
|
||||
return parsedExisting === enrichedTitle && existingTitle !== enrichedTitle ? enrichedTitle : existingTitle;
|
||||
}
|
||||
|
||||
function canReplaceExistingCover(existingCoverPath: string, previousProvenance: MetadataProvenance): boolean {
|
||||
const provenance = previousProvenance.coverPath;
|
||||
return (provenance == null || provenance === "local" || provenance === "existing") && isLocalCoverPath(existingCoverPath);
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { BookMetadata } from "../scanner/metadata.js";
|
||||
|
||||
export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf";
|
||||
export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf" | "mangadex" | "comicvine";
|
||||
|
||||
export type BookIdentifiers = {
|
||||
isbn10: string | null;
|
||||
@ -40,9 +40,17 @@ export type MetadataSearchQuery = {
|
||||
|
||||
export type MetadataMatch = Partial<BookMetadata> & {
|
||||
sourceId?: string | null;
|
||||
coverUrl?: string | null;
|
||||
identifiers?: Partial<BookIdentifiers>;
|
||||
scoreTitle?: string | null;
|
||||
};
|
||||
|
||||
export type MetadataField = "title" | "author" | "description" | "isbn" | "language" | "publisher" | "publishedDate" | "coverPath";
|
||||
|
||||
export type MetadataStatus = "enriched" | "partial" | "none";
|
||||
|
||||
export type MetadataProvenance = Partial<Record<string, MetadataProviderId | "existing" | string>>;
|
||||
|
||||
export type MetadataProviderConfig = {
|
||||
provider: MetadataProviderId;
|
||||
enabled: boolean;
|
||||
|
||||
23
apps/api/src/metadata/normalize-published-date.test.ts
Normal file
23
apps/api/src/metadata/normalize-published-date.test.ts
Normal file
@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizePublishedDate } from "./use-cases/normalize-published-date.js";
|
||||
|
||||
describe("normalizePublishedDate", () => {
|
||||
it("rejects sentinel and absurd dates seen in real metadata providers", () => {
|
||||
expect(normalizePublishedDate("0101-01-01T00:00:00+00:00")).toBeNull();
|
||||
expect(normalizePublishedDate("0001-01-01")).toBeNull();
|
||||
expect(normalizePublishedDate("1970-01-01")).toBeNull();
|
||||
expect(normalizePublishedDate("0000")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps only credible supported date formats", () => {
|
||||
expect(normalizePublishedDate("2007")).toBe("2007");
|
||||
expect(normalizePublishedDate("2007-07")).toBe("2007-07");
|
||||
expect(normalizePublishedDate("2007-07-21")).toBe("2007-07-21");
|
||||
expect(normalizePublishedDate("2007-07-21T00:00:00+00:00")).toBe("2007-07-21");
|
||||
});
|
||||
|
||||
it("rejects years outside the supported publication range", () => {
|
||||
expect(normalizePublishedDate("1499")).toBeNull();
|
||||
expect(normalizePublishedDate("2028")).toBeNull();
|
||||
});
|
||||
});
|
||||
44
apps/api/src/metadata/use-cases/normalize-published-date.ts
Normal file
44
apps/api/src/metadata/use-cases/normalize-published-date.ts
Normal file
@ -0,0 +1,44 @@
|
||||
const minimumYear = 1500;
|
||||
const maximumYear = 2027;
|
||||
const rejectedExactDates = new Set(["0001-01-01", "0101-01-01", "1970-01-01"]);
|
||||
|
||||
export function normalizePublishedDate(value: string | null | undefined): string | null {
|
||||
const text = value?.trim();
|
||||
if (!text) return null;
|
||||
|
||||
const isoDate = text.match(/^(\d{4})-(\d{2})-(\d{2})(?:[T\s].*)?$/);
|
||||
if (isoDate) {
|
||||
const [, year, month, day] = isoDate;
|
||||
const date = `${year}-${month}-${day}`;
|
||||
if (rejectedExactDates.has(date)) return null;
|
||||
return validDate(Number(year), Number(month), Number(day)) ? date : null;
|
||||
}
|
||||
|
||||
const yearMonth = text.match(/^(\d{4})-(\d{2})$/);
|
||||
if (yearMonth) {
|
||||
const [, year, month] = yearMonth;
|
||||
return validYear(Number(year)) && validMonth(Number(month)) ? `${year}-${month}` : null;
|
||||
}
|
||||
|
||||
const yearOnly = text.match(/^(\d{4})$/);
|
||||
if (yearOnly) {
|
||||
const year = Number(yearOnly[1]);
|
||||
return validYear(year) ? yearOnly[1] : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function validDate(year: number, month: number, day: number): boolean {
|
||||
if (!validYear(year) || !validMonth(month) || day < 1 || day > 31) return false;
|
||||
const date = new Date(Date.UTC(year, month - 1, day));
|
||||
return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day;
|
||||
}
|
||||
|
||||
function validYear(year: number): boolean {
|
||||
return Number.isInteger(year) && year >= minimumYear && year <= maximumYear;
|
||||
}
|
||||
|
||||
function validMonth(month: number): boolean {
|
||||
return Number.isInteger(month) && month >= 1 && month <= 12;
|
||||
}
|
||||
@ -3,56 +3,125 @@ import { MetadataMatch, MetadataSearchQuery } from "../metadata.types.js";
|
||||
export type ScoredMetadataMatch = {
|
||||
match: MetadataMatch;
|
||||
score: number;
|
||||
titleScore: number;
|
||||
authorScore: number | null;
|
||||
dateScore: number | null;
|
||||
isbnMatch: boolean;
|
||||
};
|
||||
|
||||
export class ScoreMetadataMatch {
|
||||
score(query: MetadataSearchQuery, match: MetadataMatch): number {
|
||||
let score = 0;
|
||||
const titleScore = similarity(normalize(query.title), normalize(match.title ?? ""));
|
||||
score += titleScore * 0.7;
|
||||
|
||||
if (query.author && match.author) {
|
||||
score += similarity(normalize(query.author), normalize(match.author)) * 0.2;
|
||||
} else if (!query.author) {
|
||||
score += 0.08;
|
||||
}
|
||||
|
||||
const queryYear = query.year ?? null;
|
||||
const matchYear = match.publishedDate?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
if (queryYear && matchYear) score += queryYear === matchYear ? 0.1 : -0.1;
|
||||
|
||||
if (query.isbn && match.isbn && query.isbn.replace(/\D/g, "") === match.isbn.replace(/\D/g, "")) {
|
||||
score += 0.25;
|
||||
}
|
||||
return Math.max(0, Math.min(1, score));
|
||||
return this.details(query, match).score;
|
||||
}
|
||||
|
||||
best(query: MetadataSearchQuery, matches: MetadataMatch[], minimumScore = 0.55): ScoredMetadataMatch | null {
|
||||
details(query: MetadataSearchQuery, match: MetadataMatch): ScoredMetadataMatch {
|
||||
const titleScore = scoreTitle(query.title, match.scoreTitle ?? match.title ?? "");
|
||||
const authorScore = query.author ? scoreAuthor(query.author, match.author) : null;
|
||||
const dateScore = query.year ? scoreDate(query.year, match.publishedDate) : null;
|
||||
const isbnMatch = exactIsbnMatch(query.isbn, match.isbn);
|
||||
const maxPossible = 100 + (authorScore == null ? 0 : 30) + (dateScore == null ? 0 : 10);
|
||||
const sum = titleScore + (authorScore ?? 0) + (dateScore ?? 0);
|
||||
return {
|
||||
match,
|
||||
score: maxPossible ? Math.round((100 * sum) / maxPossible) : 0,
|
||||
titleScore,
|
||||
authorScore,
|
||||
dateScore,
|
||||
isbnMatch
|
||||
};
|
||||
}
|
||||
|
||||
best(query: MetadataSearchQuery, matches: MetadataMatch[], minimumScore = 0): ScoredMetadataMatch | null {
|
||||
const scored = matches
|
||||
.map((match) => ({ match, score: this.score(query, match) }))
|
||||
.sort((left, right) => right.score - left.score);
|
||||
.map((match) => this.details(query, match))
|
||||
.sort((left, right) => compareScoredMatches(query, left, right));
|
||||
const best = scored[0];
|
||||
return best && best.score >= minimumScore ? best : null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(value: string): string {
|
||||
function compareScoredMatches(query: MetadataSearchQuery, left: ScoredMetadataMatch, right: ScoredMetadataMatch): number {
|
||||
if (left.isbnMatch !== right.isbnMatch) return left.isbnMatch ? -1 : 1;
|
||||
const leftTitleAuthor = isHighConfidenceTitleAuthor(query, left);
|
||||
const rightTitleAuthor = isHighConfidenceTitleAuthor(query, right);
|
||||
if (leftTitleAuthor !== rightTitleAuthor) return leftTitleAuthor ? -1 : 1;
|
||||
return right.score - left.score;
|
||||
}
|
||||
|
||||
function isHighConfidenceTitleAuthor(query: MetadataSearchQuery, scored: ScoredMetadataMatch): boolean {
|
||||
return scored.titleScore > 90 && (!query.author || (scored.authorScore ?? 0) >= 15);
|
||||
}
|
||||
|
||||
function scoreTitle(left: string, right: string): number {
|
||||
const normalizedLeft = normalizeTitle(left);
|
||||
const normalizedRight = normalizeTitle(right);
|
||||
if (!normalizedLeft || !normalizedRight) return 0;
|
||||
if (normalizedLeft === normalizedRight) return 100;
|
||||
if (normalizedLeft.includes(normalizedRight) || normalizedRight.includes(normalizedLeft)) return 95;
|
||||
return Math.round(jaccard(tokens(normalizedLeft), tokens(normalizedRight)) * 100);
|
||||
}
|
||||
|
||||
function scoreAuthor(localAuthor: string, sourceAuthor: string | null | undefined): number {
|
||||
const local = authorSet(localAuthor);
|
||||
if (!local.size) return 0;
|
||||
const source = authorSet(sourceAuthor ?? "");
|
||||
const present = [...local].filter((author) => source.has(author)).length;
|
||||
return 30 * (present / local.size);
|
||||
}
|
||||
|
||||
function scoreDate(localYear: string, sourceDate: string | null | undefined): number {
|
||||
const left = Number(yearFrom(localYear));
|
||||
const right = Number(yearFrom(sourceDate ?? ""));
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 10;
|
||||
return Math.abs(left - right) <= 1 ? 5 : 0;
|
||||
}
|
||||
|
||||
function exactIsbnMatch(left: string | null | undefined, right: string | null | undefined): boolean {
|
||||
const normalizedLeft = normalizeIsbn(left);
|
||||
const normalizedRight = normalizeIsbn(right);
|
||||
return Boolean(normalizedLeft && normalizedRight && normalizedLeft === normalizedRight);
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
return normalizeText(value.split(":")[0] ?? "").replace(/^(?:le|la|les|the|a|an|l)\s+/, "");
|
||||
}
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.replace(/\b(le|la|les|the|a|an|de|du|des|et|and)\b/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function similarity(left: string, right: string): number {
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
const leftTokens = new Set(left.split(" "));
|
||||
const rightTokens = new Set(right.split(" "));
|
||||
function authorSet(value: string): Set<string> {
|
||||
return new Set(
|
||||
value
|
||||
.split(/[,;&/]|\band\b|\bet\b/gi)
|
||||
.map(normalizeText)
|
||||
.filter(Boolean)
|
||||
.sort()
|
||||
);
|
||||
}
|
||||
|
||||
function tokens(value: string): Set<string> {
|
||||
return new Set(value.split(" ").filter(Boolean));
|
||||
}
|
||||
|
||||
function jaccard(leftTokens: Set<string>, rightTokens: Set<string>): number {
|
||||
const intersection = [...leftTokens].filter((token) => rightTokens.has(token)).length;
|
||||
const union = new Set([...leftTokens, ...rightTokens]).size;
|
||||
return union ? intersection / union : 0;
|
||||
}
|
||||
|
||||
function yearFrom(value: string): string | null {
|
||||
return value.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function normalizeIsbn(value: string | null | undefined): string | null {
|
||||
const normalized = value?.replace(/[^0-9X]/gi, "").toUpperCase() ?? "";
|
||||
return normalized || null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user