diff --git a/apps/api/src/common/cbr.ts b/apps/api/src/common/cbr.ts index cbeacad..76f44d2 100644 --- a/apps/api/src/common/cbr.ts +++ b/apps/api/src/common/cbr.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { extname, join } from "node:path"; import { createExtractorFromFile } from "node-unrar-js"; import { COMIC_IMAGE_EXTENSIONS, MAX_COMIC_ARCHIVE_ENTRIES } from "./cbz.js"; @@ -46,6 +46,7 @@ export async function readCbrPage( throw new Error("CBR page not found"); } + mkdirSync(storageDir, { recursive: true }); const tempDir = mkdtempSync(join(storageDir, "cbr-page-")); const safeName = `page${extname(page.entryName).toLowerCase() || ".jpg"}`; try { diff --git a/apps/api/src/metadata/adapters/bnf.provider.ts b/apps/api/src/metadata/adapters/bnf.provider.ts index 575d20d..bf35efa 100644 --- a/apps/api/src/metadata/adapters/bnf.provider.ts +++ b/apps/api/src/metadata/adapters/bnf.provider.ts @@ -2,6 +2,8 @@ import { Injectable } from "@nestjs/common"; import { XMLParser } from "fast-xml-parser"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; import { toIsbn13 } from "../use-cases/extract-identifiers.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; +import { providerFetch, providerHttpError } from "./provider-fetch.js"; const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true }); @@ -36,8 +38,8 @@ export class BnfProvider implements MetadataProvider { url.searchParams.set("query", query); url.searchParams.set("maximumRecords", String(maximumRecords)); - const response = await fetch(url, { signal: AbortSignal.timeout(5000) }); - if (!response.ok) return []; + const response = await providerFetch(this.id, url, { timeoutMs: 5000 }); + if (!response.ok) throw await providerHttpError(this.id, response, `BnF HTTP ${response.status}`); const parsed = parser.parse(await response.text()); const records = asArray(parsed?.searchRetrieveResponse?.records?.record) .map((entry) => (entry.recordData as Record | undefined)?.record) @@ -53,7 +55,7 @@ export class BnfProvider implements MetadataProvider { isbn: bestIsbn(fields, expectedIsbn13), language: subfield(fields, "101", "a"), publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"), - publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d")) + publishedDate: normalizePublishedDate(cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))) }; }) .sort((left, right) => Number(Boolean(right.isbn)) - Number(Boolean(left.isbn))); diff --git a/apps/api/src/metadata/adapters/comic-vine.provider.ts b/apps/api/src/metadata/adapters/comic-vine.provider.ts new file mode 100644 index 0000000..bbbce6f --- /dev/null +++ b/apps/api/src/metadata/adapters/comic-vine.provider.ts @@ -0,0 +1,160 @@ +import { Injectable } from "@nestjs/common"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; +import { providerFetch } from "./provider-fetch.js"; + +@Injectable() +export class ComicVineProvider implements MetadataProvider { + readonly id = "comicvine" as const; + + async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise { + const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config); + return matches[0] ?? null; + } + + async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise { + assertApiKey(config); + const title = cleanComicTitle(query.title); + const relaxed = title.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim(); + const matches = [ + ...(await searchComicVine("volume", title, config)), + ...(await searchComicVine("issue", title, config)), + ...(relaxed && relaxed !== title ? await searchComicVine("volume", relaxed, config) : []) + ]; + return rankMatches(query.title, dedupe(matches)); + } +} + +export class ComicVineProviderError extends Error { + constructor( + readonly code: "missing-key" | "invalid-key" | "rate-limit" | "http", + readonly status: number, + message: string + ) { + super(message); + this.name = "ComicVineProviderError"; + } +} + +async function searchComicVine(resource: "volume" | "issue", title: string, config: MetadataProviderConfig): Promise { + const url = new URL("https://comicvine.gamespot.com/api/search/"); + url.searchParams.set("api_key", config.apiKey!); + url.searchParams.set("format", "json"); + url.searchParams.set("resources", resource); + url.searchParams.set("query", title); + url.searchParams.set("limit", "10"); + url.searchParams.set( + "field_list", + resource === "volume" ? "id,name,description,image,start_year,publisher" : "id,name,description,image,cover_date,store_date,volume" + ); + const response = await providerFetch("comicvine", url, { + headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (Comic Vine; non-commercial)" }, + timeoutMs: 6000 + }); + const data = (await parseComicVineResponse(response)) as { results?: Array> }; + return (data.results ?? []).map((entry) => comicVineToMatch(resource, entry)); +} + +function assertApiKey(config: MetadataProviderConfig): void { + if (!config.apiKey?.trim()) { + throw new ComicVineProviderError("missing-key", 0, "Comic Vine API key is required"); + } +} + +async function parseComicVineResponse(response: Response): Promise { + const data = (await response.json().catch(() => ({}))) as { status_code?: number; error?: string }; + if (response.status === 429) throw new ComicVineProviderError("rate-limit", response.status, data.error ?? "Comic Vine rate limit"); + if (response.status === 401 || response.status === 403) throw new ComicVineProviderError("invalid-key", response.status, data.error ?? "Comic Vine API key rejected"); + if (!response.ok) throw new ComicVineProviderError("http", response.status, data.error ?? `Comic Vine HTTP ${response.status}`); + if (data.status_code && data.status_code !== 1) { + const code = data.status_code === 100 || data.status_code === 101 ? "invalid-key" : "http"; + throw new ComicVineProviderError(code, 200, data.error ?? `Comic Vine status ${data.status_code}`); + } + return data; +} + +function comicVineToMatch(resource: "volume" | "issue", entry: Record): MetadataMatch & { comicVineRank?: number } { + const volume = objectValue(entry.volume); + const title = resource === "issue" ? [stringValue(volume.name), stringValue(entry.name)].filter(Boolean).join(" ") : stringValue(entry.name); + return { + title: title || undefined, + description: cleanHtml(stringValue(entry.description)), + publisher: stringValue(objectValue(entry.publisher).name), + publishedDate: normalizePublishedDate(resource === "volume" ? stringValue(entry.start_year) : yearFromDate(stringValue(entry.cover_date) ?? stringValue(entry.store_date))), + coverUrl: imageUrl(entry.image), + sourceId: stringValue(entry.id) + }; +} + +function cleanComicTitle(value: string): string { + return value + .replace(/\.[A-Za-z0-9]{2,5}$/g, " ") + .replace(/[._]+/g, " ") + .replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ") + .replace(/\([^)]*\)/g, " ") + .replace(/\b(e?bdz|Paprika\+?|emuleCenter(?:\.|\s+)net)\b/gi, " ") + .replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ") + .replace(/[+]+/g, " ") + .replace(/\s+-\s+/g, " ") + .replace(/\s*-\s*$/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function rankMatches(originalTitle: string, matches: Array): MetadataMatch[] { + return [...matches] + .map((match) => ({ ...match, comicVineRank: comicRank(originalTitle, match) })) + .sort((left, right) => (right.comicVineRank ?? 0) - (left.comicVineRank ?? 0)) + .map(({ comicVineRank: _rank, ...match }) => match); +} + +function comicRank(originalTitle: string, match: MetadataMatch): number { + let rank = tokenOverlap(cleanComicTitle(originalTitle), match.title ?? "") * 10; + if (match.coverUrl) rank += 1; + if (match.description) rank += 1; + return rank; +} + +function dedupe(matches: MetadataMatch[]): MetadataMatch[] { + const seen = new Set(); + return matches.filter((match) => { + const key = [match.sourceId, match.title].filter(Boolean).join("|"); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function cleanHtml(value: string | null): string | null { + if (!value) return null; + return value.replace(/<[^>]*>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/\s+/g, " ").trim() || null; +} + +function imageUrl(value: unknown): string | null { + const image = objectValue(value); + return stringValue(image.original_url) ?? stringValue(image.super_url) ?? stringValue(image.medium_url) ?? stringValue(image.small_url); +} + +function yearFromDate(value: string | null): string | null { + return value?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null; +} + +function tokenOverlap(left: string, right: string): number { + const leftTokens = new Set(normalizeTokens(left)); + const rightTokens = new Set(normalizeTokens(right)); + if (!leftTokens.size || !rightTokens.size) return 0; + return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size; +} + +function normalizeTokens(value: string): string[] { + return value.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(" ").filter(Boolean); +} + +function objectValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stringValue(value: unknown): string | null { + if (typeof value === "number") return String(value); + return typeof value === "string" && value.trim() ? value.trim() : null; +} diff --git a/apps/api/src/metadata/adapters/google-books.provider.ts b/apps/api/src/metadata/adapters/google-books.provider.ts index 793a1d8..1fa6d74 100644 --- a/apps/api/src/metadata/adapters/google-books.provider.ts +++ b/apps/api/src/metadata/adapters/google-books.provider.ts @@ -1,6 +1,8 @@ import { Injectable } from "@nestjs/common"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; import { toIsbn13 } from "../use-cases/extract-identifiers.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; +import { providerFetch } from "./provider-fetch.js"; @Injectable() export class GoogleBooksProvider implements MetadataProvider { @@ -12,54 +14,157 @@ export class GoogleBooksProvider implements MetadataProvider { const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config); return matches[0] ?? null; } - const query = `isbn:${isbn}`; - const url = new URL("https://www.googleapis.com/books/v1/volumes"); - url.searchParams.set("q", query); - url.searchParams.set("maxResults", "1"); - url.searchParams.set("printType", "books"); - if (config.apiKey) url.searchParams.set("key", config.apiKey); - - const response = await fetch(url, { signal: AbortSignal.timeout(4000) }); - if (!response.ok) return null; - const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record }> }; - const info = data.items?.[0]?.volumeInfo; - if (!info) return null; - return { - title: stringValue(info.title) ?? undefined, - author: arrayJoin(info.authors), - description: stringValue(info.description), - language: stringValue(info.language), - publisher: stringValue(info.publisher), - publishedDate: stringValue(info.publishedDate), - isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13) - }; + const matches = await this.searchVolumes(`isbn:${isbn}`, config, lookup.title, lookup.author, lookup.identifiers.isbn13); + return rankMatches(lookup.title, matches)[0] ?? null; } async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise { + const attempts = googleBookQueries(query); + const matches: MetadataMatch[] = []; + const seen = new Set(); + for (const attempt of attempts) { + for (const match of await this.searchVolumes(attempt, config, query.title, query.author, query.isbn ? toIsbn13(query.isbn) : null)) { + const key = [match.sourceId, match.isbn, match.title, match.author].filter(Boolean).join("|"); + if (seen.has(key)) continue; + seen.add(key); + matches.push(match); + } + } + return rankMatches(query.title, matches); + } + + private async searchVolumes( + googleQuery: string, + config: MetadataProviderConfig, + originalTitle: string, + originalAuthor: string | null, + expectedIsbn13: string | null + ): Promise { const url = new URL("https://www.googleapis.com/books/v1/volumes"); - url.searchParams.set("q", `intitle:${query.title}${query.author ? `+inauthor:${query.author}` : ""}`); - url.searchParams.set("maxResults", "5"); + url.searchParams.set("q", googleQuery); + url.searchParams.set("maxResults", "10"); url.searchParams.set("printType", "books"); if (config.apiKey) url.searchParams.set("key", config.apiKey); - const response = await fetch(url, { signal: AbortSignal.timeout(4000) }); - if (!response.ok) return []; - const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record }> }; + const response = await providerFetch(this.id, url, { timeoutMs: 4000 }); + const data = (await parseGoogleResponse(response)) as { items?: Array<{ id?: string; volumeInfo?: Record }> }; return (data.items ?? []) - .map((item) => item.volumeInfo) - .filter((info): info is Record => Boolean(info)) - .map((info) => ({ + .map((item) => ({ sourceId: item.id, info: item.volumeInfo })) + .filter((item): item is { sourceId: string | undefined; info: Record } => Boolean(item.info)) + .map(({ sourceId, info }) => ({ title: stringValue(info.title) ?? undefined, author: arrayJoin(info.authors), description: stringValue(info.description), language: stringValue(info.language), publisher: stringValue(info.publisher), - publishedDate: stringValue(info.publishedDate), - isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, query.isbn ? toIsbn13(query.isbn) : null) + publishedDate: normalizePublishedDate(stringValue(info.publishedDate)), + isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, expectedIsbn13), + coverUrl: coverUrl(info.imageLinks), + sourceId, + identifiers: { candidates: isbnCandidates(info.industryIdentifiers) }, + googleRank: googleRank(originalTitle, originalAuthor, info) })); } } +export class GoogleBooksProviderError extends Error { + constructor( + readonly code: "quota" | "auth" | "http", + readonly status: number, + message: string + ) { + super(message); + this.name = "GoogleBooksProviderError"; + } +} + +async function parseGoogleResponse(response: Response): Promise { + const data = (await response.json().catch(() => ({}))) as { error?: { message?: string; status?: string } }; + if (response.ok) return data; + const message = data.error?.message ?? `Google Books HTTP ${response.status}`; + if (response.status === 429) throw new GoogleBooksProviderError("quota", response.status, message); + if (response.status === 401 || response.status === 403) throw new GoogleBooksProviderError("auth", response.status, message); + throw new GoogleBooksProviderError("http", response.status, message); +} + +function googleBookQueries(query: MetadataSearchQuery): string[] { + const cleaned = cleanGoogleBooksTitle(query.title); + const relaxed = relaxSeriesTitle(cleaned); + return [ + query.isbn ? `isbn:${query.isbn}` : null, + googleTitleQuery(cleaned, query.author, true), + googleTitleQuery(cleaned, query.author, false), + relaxed !== cleaned ? googleTitleQuery(relaxed, query.author, true) : null, + relaxed !== cleaned ? googleTitleQuery(relaxed, null, false) : null, + googleTitleQuery(cleaned, null, false) + ].filter((value, index, values): value is string => Boolean(value) && values.indexOf(value) === index); +} + +function googleTitleQuery(title: string, author: string | null, quoted: boolean): string { + const titlePart = quoted ? `intitle:"${title.replace(/"/g, " ")}"` : `intitle:${title}`; + return author ? `${titlePart}+inauthor:${author}` : titlePart; +} + +function cleanGoogleBooksTitle(value: string): string { + return value + .replace(/\.[A-Za-z0-9]{2,5}$/g, " ") + .replace(/[._]+/g, " ") + .replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ") + .replace(/\b(e?bdz|Paprika\+?|emuleCenter\.net)\b/gi, " ") + .replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ") + .replace(/\bVol(?:ume)?\.?\s*0?(\d{1,3})\b/gi, " $1 ") + .replace(/[+]+/g, " ") + .replace(/\s+-\s+/g, " ") + .replace(/\s*-\s*$/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function relaxSeriesTitle(value: string): string { + return value.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim(); +} + +function rankMatches(originalTitle: string, matches: Array): MetadataMatch[] { + return [...matches] + .sort((left, right) => (right.googleRank ?? 0) - (left.googleRank ?? 0)) + .map(({ googleRank: _googleRank, ...match }) => match); +} + +function googleRank(originalTitle: string, originalAuthor: string | null, info: Record): number { + const expectedVolume = volumeNumber(originalTitle); + const candidateTitle = [stringValue(info.title), stringValue(info.subtitle)].filter(Boolean).join(" "); + let rank = tokenOverlap(cleanGoogleBooksTitle(originalTitle), candidateTitle) * 10; + if (expectedVolume) { + const candidateVolume = volumeNumber(candidateTitle); + rank += candidateVolume === expectedVolume ? 6 : candidateVolume ? -4 : 0; + } + if (originalAuthor && arrayJoin(info.authors)?.toLowerCase().includes(originalAuthor.toLowerCase())) rank += 2; + if (stringValue(info.description)) rank += 1; + if (coverUrl(info.imageLinks)) rank += 1; + return rank; +} + +function volumeNumber(value: string): string | null { + return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null; +} + +function tokenOverlap(left: string, right: string): number { + const leftTokens = new Set(normalizeTokens(left)); + const rightTokens = new Set(normalizeTokens(right)); + if (!leftTokens.size || !rightTokens.size) return 0; + return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size; +} + +function normalizeTokens(value: string): string[] { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .split(" ") + .filter(Boolean); +} + function stringValue(value: unknown): string | null { return typeof value === "string" && value.trim() ? value.trim() : null; } @@ -77,3 +182,20 @@ function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | nu const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier; return stringValue(isbn13) ?? stringValue(isbn10); } + +function isbnCandidates(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.map((entry) => stringValue((entry as { identifier?: unknown }).identifier)).filter((entry): entry is string => Boolean(entry)); +} + +function coverUrl(value: unknown): string | null { + if (!value || typeof value !== "object") return null; + const links = value as Record; + return ( + stringValue(links.extraLarge) ?? + stringValue(links.large) ?? + stringValue(links.medium) ?? + stringValue(links.thumbnail) ?? + stringValue(links.smallThumbnail) + )?.replace(/^http:/, "https:") ?? null; +} diff --git a/apps/api/src/metadata/adapters/local.provider.ts b/apps/api/src/metadata/adapters/local.provider.ts index 64061cf..8ac1762 100644 --- a/apps/api/src/metadata/adapters/local.provider.ts +++ b/apps/api/src/metadata/adapters/local.provider.ts @@ -1,5 +1,6 @@ import { Injectable } from "@nestjs/common"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; @Injectable() export class LocalMetadataProvider implements MetadataProvider { @@ -20,7 +21,7 @@ export class LocalMetadataProvider implements MetadataProvider { title: query.title, author: query.author, isbn: query.isbn ?? null, - publishedDate: query.year ?? null + publishedDate: normalizePublishedDate(query.year) } ]; } diff --git a/apps/api/src/metadata/adapters/mangadex.provider.ts b/apps/api/src/metadata/adapters/mangadex.provider.ts new file mode 100644 index 0000000..e8b3b2c --- /dev/null +++ b/apps/api/src/metadata/adapters/mangadex.provider.ts @@ -0,0 +1,179 @@ +import { Injectable } from "@nestjs/common"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; +import { extractSeriesVolume } from "../use-cases/extract-series-volume.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; +import { providerFetch } from "./provider-fetch.js"; + +@Injectable() +export class MangaDexProvider implements MetadataProvider { + readonly id = "mangadex" as const; + + async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise { + const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config); + return matches[0] ?? null; + } + + async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise { + const titles = mangaDexTitleQueries(query.title); + const matches: Array = []; + const seen = new Map(); + for (const title of titles) { + for (const match of await searchManga(title, config)) { + const key = match.sourceId ?? `${match.title}|${match.author}`; + const ranked = { ...match, scoreTitle: title, mangaDexRank: mangaRank(query.title, title, match) }; + const existingIndex = seen.get(key); + if (existingIndex === undefined) { + seen.set(key, matches.length); + matches.push(ranked); + continue; + } + if ((ranked.mangaDexRank ?? 0) > (matches[existingIndex]?.mangaDexRank ?? 0)) { + matches[existingIndex] = ranked; + } + } + } + return rankMatches(matches); + } +} + +export class MangaDexProviderError extends Error { + constructor( + readonly code: "rate-limit" | "http", + readonly status: number, + message: string + ) { + super(message); + this.name = "MangaDexProviderError"; + } +} + +async function searchManga(title: string, _config: MetadataProviderConfig): Promise { + const url = new URL("https://api.mangadex.org/manga"); + url.searchParams.set("title", title); + url.searchParams.set("limit", "10"); + url.searchParams.set("includes[]", "cover_art"); + url.searchParams.append("includes[]", "author"); + url.searchParams.append("includes[]", "artist"); + url.searchParams.set("contentRating[]", "safe"); + url.searchParams.append("contentRating[]", "suggestive"); + let response = await providerFetch("mangadex", url, { + headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" }, + timeoutMs: 5000 + }); + if (response.status === 429) { + await sleep(retryDelayMs(response)); + response = await providerFetch("mangadex", url, { + headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" }, + timeoutMs: 5000 + }); + } + const data = (await parseMangaDexResponse(response)) as { data?: Array> }; + return (data.data ?? []).map(mangaToMatch); +} + +function retryDelayMs(response: Response): number { + const retryAfter = Number(response.headers.get("Retry-After")); + return Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter * 1000, 2000) : 250; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function parseMangaDexResponse(response: Response): Promise { + const data = (await response.json().catch(() => ({}))) as { errors?: Array<{ detail?: string; title?: string }> }; + if (response.ok) return data; + const message = data.errors?.map((error) => error.detail ?? error.title).filter(Boolean).join("; ") || `MangaDex HTTP ${response.status}`; + if (response.status === 429) throw new MangaDexProviderError("rate-limit", response.status, message); + throw new MangaDexProviderError("http", response.status, message); +} + +function mangaToMatch(manga: Record): MetadataMatch & { mangaDexRank?: number } { + const id = stringValue(manga.id); + const attributes = objectValue(manga.attributes); + const relationships = Array.isArray(manga.relationships) ? (manga.relationships as Array>) : []; + const cover = relationships.find((entry) => entry.type === "cover_art"); + const coverFile = stringValue(objectValue(cover?.attributes).fileName); + return { + title: localizedText(attributes.title) ?? undefined, + author: relationshipNames(relationships), + description: localizedText(attributes.description), + publishedDate: normalizePublishedDate(stringValue(attributes.year)), + language: stringValue(attributes.originalLanguage), + coverUrl: id && coverFile ? `https://uploads.mangadex.org/covers/${id}/${coverFile}.512.jpg` : null, + sourceId: id + }; +} + +function mangaDexTitleQueries(title: string): string[] { + const cleaned = extractSeriesVolume(title).seriesTitle; + return [cleaned, ...mangaDexTitleAliases(cleaned)].filter( + (value, index, values): value is string => Boolean(value) && values.indexOf(value) === index + ); +} + +function mangaDexTitleAliases(title: string): string[] { + const normalized = normalizeTitle(title); + if (normalized === "demon slayer school days") return ["Demon Slayer Kimetsu Academy", "Kimetsu Academy"]; + return []; +} + +function mangaRank(originalTitle: string, searchedTitle: string, match: MetadataMatch): number { + const expectedVolume = volumeNumber(originalTitle); + let rank = tokenOverlap(searchedTitle, match.title ?? "") * 10; + if (expectedVolume) { + const candidateVolume = volumeNumber(match.title ?? ""); + rank += candidateVolume === expectedVolume ? 4 : candidateVolume ? -2 : 0; + } + if (match.coverUrl) rank += 1; + if (match.description) rank += 1; + return rank; +} + +function rankMatches(matches: Array): MetadataMatch[] { + return [...matches].sort((left, right) => (right.mangaDexRank ?? 0) - (left.mangaDexRank ?? 0)).map(({ mangaDexRank: _rank, ...match }) => match); +} + +function volumeNumber(value: string): string | null { + return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null; +} + +function tokenOverlap(left: string, right: string): number { + const leftTokens = new Set(normalizeTitle(left).split(" ").filter(Boolean)); + const rightTokens = new Set(normalizeTitle(right).split(" ").filter(Boolean)); + if (!leftTokens.size || !rightTokens.size) return 0; + return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size; +} + +function normalizeTitle(value: string): string { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function localizedText(value: unknown): string | null { + if (!value || typeof value !== "object") return null; + const entries = value as Record; + return stringValue(entries.en) ?? stringValue(entries.fr) ?? Object.values(entries).map(stringValue).find(Boolean) ?? null; +} + +function relationshipNames(relationships: Array>): string | null { + const names = relationships + .filter((entry) => entry.type === "author" || entry.type === "artist") + .map((entry) => stringValue(objectValue(entry.attributes).name)) + .filter((entry): entry is string => Boolean(entry)); + return names.length ? [...new Set(names)].join(", ") : null; +} + +function objectValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stringValue(value: unknown): string | null { + if (typeof value === "number") return String(value); + return typeof value === "string" && value.trim() ? value.trim() : null; +} diff --git a/apps/api/src/metadata/adapters/open-library.provider.ts b/apps/api/src/metadata/adapters/open-library.provider.ts index 892421f..2683b16 100644 --- a/apps/api/src/metadata/adapters/open-library.provider.ts +++ b/apps/api/src/metadata/adapters/open-library.provider.ts @@ -1,6 +1,8 @@ import { Injectable } from "@nestjs/common"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js"; import { toIsbn13 } from "../use-cases/extract-identifiers.js"; +import { normalizePublishedDate } from "../use-cases/normalize-published-date.js"; +import { providerFetch, providerHttpError } from "./provider-fetch.js"; @Injectable() export class OpenLibraryProvider implements MetadataProvider { @@ -24,11 +26,11 @@ export class OpenLibraryProvider implements MetadataProvider { if (query.author) url.searchParams.set("author", query.author); if (query.year) url.searchParams.set("first_publish_year", query.year); url.searchParams.set("limit", "5"); - const response = await fetch(url, { + const response = await providerFetch(this.id, url, { headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, - signal: AbortSignal.timeout(4000) + timeoutMs: 4000 }); - if (!response.ok) return []; + if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`); const data = (await response.json()) as { docs?: Array> }; return (data.docs ?? []).map((doc) => ({ title: stringValue(doc.title) ?? undefined, @@ -36,28 +38,31 @@ export class OpenLibraryProvider implements MetadataProvider { author: arrayJoin(doc.author_name), language: firstArrayValue(doc.language), publisher: firstArrayValue(doc.publisher), - publishedDate: String(doc.first_publish_year ?? "") || null, - isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null) + publishedDate: normalizePublishedDate(String(doc.first_publish_year ?? "") || null), + isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null), + coverUrl: openLibraryCoverUrl(doc.cover_i, firstArrayValue(doc.edition_key) ?? stringValue(doc.cover_edition_key)) })); } private async lookupEdition(sourceId: string, expectedIsbn13: string | null): Promise { const editionKey = sourceId.replace(/^\/?books\//, ""); if (!editionKey) return null; - const response = await fetch(`https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, { + const response = await providerFetch(this.id, `https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, { headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, - signal: AbortSignal.timeout(4000) + timeoutMs: 4000 }); - if (!response.ok) return null; + if (response.status === 404) return null; + if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`); return this.editionToMatch((await response.json()) as Record, expectedIsbn13); } private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise { - const response = await fetch(`https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, { + const response = await providerFetch(this.id, `https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, { headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, - signal: AbortSignal.timeout(4000) + timeoutMs: 4000 }); - if (!response.ok) return null; + if (response.status === 404) return null; + if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`); return this.editionToMatch((await response.json()) as Record, expectedIsbn13); } @@ -70,18 +75,20 @@ export class OpenLibraryProvider implements MetadataProvider { isbn: bestIsbn([...(asStringArray(edition.isbn_13)), ...(asStringArray(edition.isbn_10))], expectedIsbn13), language: languageValue(edition.languages), publisher: firstArrayValue(edition.publishers), - publishedDate: stringValue(edition.publish_date) + publishedDate: normalizePublishedDate(stringValue(edition.publish_date)), + coverUrl: editionCoverUrl(edition) }; } private async lookupAuthorName(value: unknown): Promise { const key = (Array.isArray(value) ? value[0] : undefined)?.key; if (typeof key !== "string") return null; - const response = await fetch(`https://openlibrary.org${key}.json`, { + const response = await providerFetch(this.id, `https://openlibrary.org${key}.json`, { headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, - signal: AbortSignal.timeout(3000) + timeoutMs: 3000 }); - if (!response.ok) return null; + if (response.status === 404) return null; + if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`); const author = (await response.json()) as Record; return stringValue(author.name); } @@ -124,3 +131,18 @@ function languageValue(value: unknown): string | null { const key = (Array.isArray(value) ? value[0] : undefined)?.key; return typeof key === "string" ? key.split("/").pop() ?? null : null; } + +function openLibraryCoverUrl(coverId: unknown, editionKey: string | null): string | null { + if (typeof coverId === "number" || typeof coverId === "string") { + return `https://covers.openlibrary.org/b/id/${encodeURIComponent(String(coverId))}-L.jpg`; + } + if (editionKey) { + return `https://covers.openlibrary.org/b/olid/${encodeURIComponent(editionKey)}-L.jpg`; + } + return null; +} + +function editionCoverUrl(edition: Record): string | null { + const covers = Array.isArray(edition.covers) ? edition.covers : []; + return openLibraryCoverUrl(covers[0], stringValue(edition.key)?.split("/").pop() ?? null); +} diff --git a/apps/api/src/metadata/adapters/provider-fetch.ts b/apps/api/src/metadata/adapters/provider-fetch.ts new file mode 100644 index 0000000..f26c5fa --- /dev/null +++ b/apps/api/src/metadata/adapters/provider-fetch.ts @@ -0,0 +1,88 @@ +import { MetadataProviderId } from "../metadata.types.js"; + +export type MetadataProviderFailureCode = "timeout" | "dns" | "quota" | "auth" | "http" | "network"; + +export class MetadataProviderRequestError extends Error { + constructor( + readonly provider: MetadataProviderId | "cover", + readonly code: MetadataProviderFailureCode, + readonly message: string, + readonly status?: number + ) { + super(message); + this.name = "MetadataProviderRequestError"; + } +} + +export async function providerFetch( + provider: MetadataProviderId | "cover", + input: string | URL, + init: RequestInit & { timeoutMs: number } +): Promise { + const { timeoutMs, ...requestInit } = init; + try { + return await fetch(input, { + ...requestInit, + signal: requestInit.signal ?? AbortSignal.timeout(timeoutMs) + }); + } catch (error) { + throw classifyFetchError(provider, error, timeoutMs); + } +} + +export async function providerHttpError( + provider: MetadataProviderId | "cover", + response: Response, + fallbackMessage: string +): Promise { + const message = (await response.text().catch(() => "")) || fallbackMessage; + if (response.status === 429) return new MetadataProviderRequestError(provider, "quota", message, response.status); + if (response.status === 401 || response.status === 403) return new MetadataProviderRequestError(provider, "auth", message, response.status); + return new MetadataProviderRequestError(provider, "http", message, response.status); +} + +export function describeMetadataProviderError(error: unknown): string { + if (error instanceof MetadataProviderRequestError) { + const status = error.status ? ` HTTP ${error.status}` : ""; + return `${error.code}${status}: ${error.message}`; + } + if (hasProviderErrorCode(error)) { + const status = typeof error.status === "number" ? ` HTTP ${error.status}` : ""; + return `${String(error.code)}${status}: ${errorMessage(error)}`; + } + return errorMessage(error); +} + +function classifyFetchError(provider: MetadataProviderId | "cover", error: unknown, timeoutMs: number): MetadataProviderRequestError { + const code = nestedCode(error); + if (isTimeoutError(error)) { + return new MetadataProviderRequestError(provider, "timeout", `request timed out after ${timeoutMs}ms`); + } + if (code === "EAI_AGAIN" || code === "ENOTFOUND") { + return new MetadataProviderRequestError(provider, "dns", code); + } + return new MetadataProviderRequestError(provider, "network", errorMessage(error)); +} + +function isTimeoutError(error: unknown): boolean { + return ( + error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError") || + error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError") + ); +} + +function nestedCode(error: unknown): string | null { + if (!error || typeof error !== "object") return null; + const direct = "code" in error && typeof error.code === "string" ? error.code : null; + if (direct) return direct; + const cause = "cause" in error ? error.cause : null; + return cause && typeof cause === "object" && "code" in cause && typeof cause.code === "string" ? cause.code : null; +} + +function hasProviderErrorCode(error: unknown): error is { code: string; status?: number; message?: string } { + return Boolean(error && typeof error === "object" && "code" in error && typeof error.code === "string"); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/apps/api/src/metadata/local-metadata-hints.test.ts b/apps/api/src/metadata/local-metadata-hints.test.ts index 19f9e2b..53bab86 100644 --- a/apps/api/src/metadata/local-metadata-hints.test.ts +++ b/apps/api/src/metadata/local-metadata-hints.test.ts @@ -38,6 +38,76 @@ describe("metadata match scoring", () => { ); expect(best?.match.title).toBe("Harry Potter et le Prince de sang-mêlé"); - expect(best?.score).toBeGreaterThan(0.7); + expect(best?.score).toBeGreaterThan(70); + }); + + it("scores titles, authors and dates with the contract weights", () => { + const scorer = new ScoreMetadataMatch(); + const result = scorer.details( + { title: "The Harry Potter et le prince de sang mêlé: édition collector", author: "J. K. Rowling", year: "2005" }, + { + title: "Harry Potter et le prince de sang-mêlé", + author: "J.K. Rowling", + publishedDate: "2006" + } + ); + + expect(result.titleScore).toBe(100); + expect(result.authorScore).toBe(30); + expect(result.dateScore).toBe(5); + expect(result.score).toBe(96); + }); + + it("uses exact ISBN matches before weaker title-only candidates", () => { + const best = new ScoreMetadataMatch().best( + { title: "Daredevil", author: null, isbn: "9782809476255" }, + [ + { + title: "Daredevil", + author: "Rosemary Carter", + isbn: "9780373105601" + }, + { + title: "Daredevil by Chip Zdarsky", + author: "Chip Zdarsky", + isbn: "9782809476255" + } + ] + ); + + expect(best?.match.author).toBe("Chip Zdarsky"); + expect(best?.isbnMatch).toBe(true); + }); + + it("scores unrelated serialized or audiobook candidates from title/author/date only", () => { + const scorer = new ScoreMetadataMatch(); + const query = { title: "Harry Potter et le prince de sang mêlé", author: "J. K. Rowling" }; + const french = scorer.score(query, { + title: "Harry Potter et le prince de sang-mêlé", + author: "J. K. Rowling", + isbn: "9782070577644" + }); + const koreanVolume = scorer.score(query, { + title: "Harry Potter et le prince de sang-mêlé - Volume 1", + author: "J. K. Rowling", + publisher: "문학수첩", + isbn: "9791193790724" + }); + + expect(french).toBeGreaterThan(80); + expect(koreanVolume).toBeLessThan(french); + }); + + it("does not apply legacy audiobook penalties outside the contract", () => { + const scorer = new ScoreMetadataMatch(); + const query = { title: "Harry Potter et le prince de sang mêlé", author: "J. K. Rowling" }; + + expect( + scorer.score(query, { + title: "Harry Potter Et Le Prince De Sang-mêlé Livre Audio", + author: "J. K. Rowling", + isbn: "9782075105170" + }) + ).toBeGreaterThan(80); }); }); diff --git a/apps/api/src/metadata/metadata-providers.test.ts b/apps/api/src/metadata/metadata-providers.test.ts index da905c1..df6cc28 100644 --- a/apps/api/src/metadata/metadata-providers.test.ts +++ b/apps/api/src/metadata/metadata-providers.test.ts @@ -1,7 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { BnfProvider } from "./adapters/bnf.provider.js"; -import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; +import { ComicVineProvider } from "./adapters/comic-vine.provider.js"; +import { GoogleBooksProvider, GoogleBooksProviderError } from "./adapters/google-books.provider.js"; +import { MangaDexProvider } from "./adapters/mangadex.provider.js"; import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; +import { MetadataProviderRequestError } from "./adapters/provider-fetch.js"; import { MetadataLookup, MetadataProviderConfig } from "./metadata.types.js"; const lookup: MetadataLookup = { @@ -58,14 +61,210 @@ describe("metadata providers", () => { }); }); - it("treats Google Books quota exhaustion as a non-blocking miss", async () => { + it("reports Google Books quota exhaustion explicitly", async () => { const fetchMock = vi.fn(async () => jsonResponse({ error: { code: 429, status: "RESOURCE_EXHAUSTED" } }, 429)); vi.stubGlobal("fetch", fetchMock); - const result = await new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" }); + await expect(new GoogleBooksProvider().lookup(lookup, { ...config, provider: "googlebooks" })).rejects.toMatchObject({ + code: "quota", + status: 429 + } satisfies Partial); expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toContain("q=isbn%3A9782070612376"); - expect(result).toBeNull(); + }); + + it("classifies provider DNS failures explicitly", async () => { + const error = new TypeError("fetch failed") as Error & { cause?: { code: string } }; + error.cause = { code: "EAI_AGAIN" }; + vi.stubGlobal("fetch", vi.fn(async () => Promise.reject(error))); + + await expect(new OpenLibraryProvider().searchByMetadata({ title: "Daredevil", author: null }, config)).rejects.toMatchObject({ + code: "dns", + message: "EAI_AGAIN" + } satisfies Partial); + }); + + it.each([ + ["Demon.Slayer.School.Days.T01.FRENCH.CBZ.eBook-ebdz", "Demon Slayer School Days 1"], + ["Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", "Dragon Ball SD 1"], + ["Eyeshield.21.T01.FRENCH.CBZ.eBook-ebdz", "Eyeshield 21 1"], + ["Solo Leveling T03", "Solo Leveling 3"] + ])("cleans noisy Google Books title queries for %s", async (title, expectedCleanTitle) => { + const fetchMock = vi.fn(async () => jsonResponse({ totalItems: 0, items: [] })); + vi.stubGlobal("fetch", fetchMock); + + await new GoogleBooksProvider().searchByMetadata({ title, author: null }, { ...config, provider: "googlebooks" }); + const queries = fetchMock.mock.calls.map((call) => new URL(String((call as unknown[])[0])).searchParams.get("q") ?? ""); + + expect(queries[0]).toBe(`intitle:"${expectedCleanTitle}"`); + expect(queries.join(" ")).not.toMatch(/\b(FRENCH|CBZ|eBook|ebdz|Paprika)\b/i); + }); + + it("sorts Google Books results by matching manga volume instead of taking the first item", async () => { + const fetchMock = vi.fn(async () => + jsonResponse({ + totalItems: 2, + items: [ + { id: "volume-2", volumeInfo: { title: "Solo Leveling, Vol. 2", authors: ["Chugong"], publishedDate: "2021" } }, + { id: "volume-3", volumeInfo: { title: "Solo Leveling, Vol. 3", authors: ["Chugong"], publishedDate: "2021" } } + ] + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const results = await new GoogleBooksProvider().searchByMetadata( + { title: "Solo Leveling T03", author: null }, + { ...config, provider: "googlebooks" } + ); + + expect(results[0]).toMatchObject({ title: "Solo Leveling, Vol. 3", sourceId: "volume-3" }); + }); + + it.each([ + ["Solo Leveling T03", "Solo Leveling"], + ["Solo Leveling 003", "Solo Leveling"], + ["Eyeshield.21.T01.FRENCH.CBZ.eBook-ebdz", "Eyeshield 21"], + ["Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", "Dragon Ball SD"] + ])("queries MangaDex with the cleaned series title for %s", async (title, expectedQuery) => { + const fetchMock = vi.fn(async () => jsonResponse({ data: [] })); + vi.stubGlobal("fetch", fetchMock); + + await new MangaDexProvider().searchByMetadata({ title, author: null }, { ...config, provider: "mangadex" }); + const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0])); + + expect(firstUrl.searchParams.get("title")).toBe(expectedQuery); + }); + + it("queries MangaDex aliases and maps cover_art to a cover URL", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ data: [] })) + .mockResolvedValueOnce( + jsonResponse({ + data: [ + { + id: "manga-1", + attributes: { + title: { en: "Demon Slayer: Kimetsu Academy" }, + description: { en: "School spin-off." }, + year: 2021, + originalLanguage: "ja" + }, + relationships: [ + { type: "cover_art", attributes: { fileName: "cover.jpg" } }, + { type: "author", attributes: { name: "Natsuki Hokami" } } + ] + } + ] + }) + ) + .mockResolvedValue(jsonResponse({ data: [] })); + vi.stubGlobal("fetch", fetchMock); + + const results = await new MangaDexProvider().searchByMetadata( + { title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ.eBook-ebdz", author: null }, + { ...config, provider: "mangadex" } + ); + const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0])); + const secondUrl = new URL(String((fetchMock.mock.calls[1] as unknown[])[0])); + + expect(firstUrl.searchParams.get("title")).toBe("Demon Slayer School Days"); + expect(secondUrl.searchParams.get("title")).toBe("Demon Slayer Kimetsu Academy"); + expect(results[0]).toMatchObject({ + title: "Demon Slayer: Kimetsu Academy", + scoreTitle: "Demon Slayer Kimetsu Academy", + author: "Natsuki Hokami", + publishedDate: "2021", + coverUrl: "https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg" + }); + }); + + it("keeps Dragon Ball SD ahead of Dragon Ball for MangaDex matches", async () => { + const fetchMock = vi.fn(async () => + jsonResponse({ + data: [ + { + id: "dragon-ball", + attributes: { title: { en: "Dragon Ball" }, description: { en: "Original series." }, year: 1984, originalLanguage: "ja" }, + relationships: [] + }, + { + id: "dragon-ball-sd", + attributes: { title: { en: "Dragon Ball SD" }, description: { en: "SD spin-off." }, year: 2010, originalLanguage: "ja" }, + relationships: [] + } + ] + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const results = await new MangaDexProvider().searchByMetadata( + { title: "Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", author: null }, + { ...config, provider: "mangadex" } + ); + + expect(results[0]).toMatchObject({ title: "Dragon Ball SD", sourceId: "dragon-ball-sd" }); + }); + + it("reports MangaDex rate limits explicitly", async () => { + vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ errors: [{ detail: "Too many requests" }] }, 429))); + + await expect( + new MangaDexProvider().searchByMetadata({ title: "Solo Leveling T03", author: null }, { ...config, provider: "mangadex" }) + ).rejects.toMatchObject({ code: "rate-limit", status: 429 }); + }); + + it("requires a Comic Vine API key before querying", async () => { + await expect( + new ComicVineProvider().searchByMetadata({ title: "Wolverine Origin", author: null }, { ...config, provider: "comicvine", apiKey: null }) + ).rejects.toMatchObject({ code: "missing-key" }); + }); + + it("queries Comic Vine volumes/issues and cleans HTML descriptions", async () => { + const fetchMock = vi.fn(async () => + jsonResponse({ + status_code: 1, + results: [ + { + id: 123, + name: "Wolverine: The Origin", + description: "

Origin story & family secrets.

", + start_year: "2001", + image: { super_url: "https://comicvine.gamespot.com/a/uploads/scale_large/origin.jpg" }, + publisher: { name: "Marvel" } + } + ] + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const results = await new ComicVineProvider().searchByMetadata( + { title: "Comics.Fr.Wolverine.Origin.by.AleK.(emuleCenter.net)", author: null }, + { ...config, provider: "comicvine", apiKey: "cv-key" } + ); + const firstUrl = new URL(String((fetchMock.mock.calls[0] as unknown[])[0])); + + expect(firstUrl.searchParams.get("resources")).toBe("volume"); + expect(firstUrl.searchParams.get("query")).toBe("Comics Fr Wolverine Origin by AleK"); + expect(results[0]).toMatchObject({ + title: "Wolverine: The Origin", + description: "Origin story & family secrets.", + publishedDate: "2001", + publisher: "Marvel", + coverUrl: "https://comicvine.gamespot.com/a/uploads/scale_large/origin.jpg" + }); + }); + + it("reports Comic Vine invalid keys and rate limits explicitly", async () => { + vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ status_code: 101, error: "Invalid API Key" }))); + await expect( + new ComicVineProvider().searchByMetadata({ title: "Daredevil", author: null }, { ...config, provider: "comicvine", apiKey: "bad" }) + ).rejects.toMatchObject({ code: "invalid-key" }); + + vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ error: "Rate limited" }, 429))); + await expect( + new ComicVineProvider().searchByMetadata({ title: "Daredevil", author: null }, { ...config, provider: "comicvine", apiKey: "ok" }) + ).rejects.toMatchObject({ code: "rate-limit", status: 429 }); }); it("queries OpenLibrary by local metadata when ISBN is missing", async () => { diff --git a/apps/api/src/metadata/metadata.module.ts b/apps/api/src/metadata/metadata.module.ts index d15002a..5f7145b 100644 --- a/apps/api/src/metadata/metadata.module.ts +++ b/apps/api/src/metadata/metadata.module.ts @@ -1,14 +1,16 @@ import { Module } from "@nestjs/common"; import { DatabaseModule } from "../database/database.module.js"; import { BnfProvider } from "./adapters/bnf.provider.js"; +import { ComicVineProvider } from "./adapters/comic-vine.provider.js"; import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; import { LocalMetadataProvider } from "./adapters/local.provider.js"; +import { MangaDexProvider } from "./adapters/mangadex.provider.js"; import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; import { MetadataService } from "./metadata.service.js"; @Module({ imports: [DatabaseModule], - providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider], + providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider, MangaDexProvider, ComicVineProvider], exports: [MetadataService] }) export class MetadataModule {} diff --git a/apps/api/src/metadata/metadata.service.test.ts b/apps/api/src/metadata/metadata.service.test.ts index 0e1adea..9a7823d 100644 --- a/apps/api/src/metadata/metadata.service.test.ts +++ b/apps/api/src/metadata/metadata.service.test.ts @@ -1,9 +1,12 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import AdmZip from "adm-zip"; import { afterEach, describe, expect, it, vi } from "vitest"; import { DatabaseService } from "../database/database.service.js"; +import { books, libraries } from "../database/schema.js"; import { BookMetadata } from "../scanner/metadata.js"; +import { MetadataProviderRequestError } from "./adapters/provider-fetch.js"; import { MetadataService } from "./metadata.service.js"; import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "./metadata.types.js"; @@ -52,7 +55,9 @@ describe("MetadataService", () => { localProvider as never, openLibraryProvider as never, providerStub("googlebooks") as never, - providerStub("bnf") as never + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never ); const localMetadata: BookMetadata = { title: "Harry Potter et le Prince de Sang Mele", @@ -73,7 +78,7 @@ describe("MetadataService", () => { expect(openLibraryLookup).toHaveBeenCalledOnce(); expect(openLibraryLookup.mock.calls[0]?.[0].identifiers.isbn13).toBe("9782070612383"); expect(result).toMatchObject({ - title: "Harry Potter et le Prince de sang-mêlé", + title: "Harry Potter et le Prince de Sang Mele", isbn: "9782070612383", isbn13: "9782070612383", description: "Harry Potter découvre l'héritage du Prince de Sang-Mêlé." @@ -82,6 +87,64 @@ describe("MetadataService", () => { database.onModuleDestroy(); }); + it.runIf(canLoadBetterSqlite())("continues enrichment after a provider DNS failure and logs the failure class", async () => { + const database = createDatabase(); + database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'googlebooks'").run(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const openLibraryProvider: MetadataProvider = { + id: "openlibrary", + lookup: async () => null, + searchByMetadata: async () => { + throw new MetadataProviderRequestError("openlibrary", "dns", "EAI_AGAIN"); + } + }; + const googleProvider: MetadataProvider = { + id: "googlebooks", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Daredevil", + author: "Roy Thomas", + description: "Daredevil keeps moving even when another provider is unreachable.", + publishedDate: "2019" + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + openLibraryProvider as never, + googleProvider as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Daredevil", + author: null, + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Daredevil.cbz", + { remote: true } + ); + + expect(warn).toHaveBeenCalledWith('[metadata] Provider openlibrary failed for "Daredevil": dns: EAI_AGAIN'); + expect(result).toMatchObject({ + title: "Daredevil", + author: "Roy Thomas", + description: "Daredevil keeps moving even when another provider is unreachable." + }); + + database.onModuleDestroy(); + }); + it.runIf(canLoadBetterSqlite())("looks up details from a title search hit even when the hit has no ISBN", async () => { const database = createDatabase(); const localProvider = providerStub("local"); @@ -112,7 +175,9 @@ describe("MetadataService", () => { localProvider as never, openLibraryProvider as never, providerStub("googlebooks") as never, - providerStub("bnf") as never + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never ); const localMetadata: BookMetadata = { title: "Harry Potter et le prince de sang mele", @@ -133,7 +198,7 @@ describe("MetadataService", () => { expect(openLibraryLookup).toHaveBeenCalledOnce(); expect(openLibraryLookup.mock.calls[0]?.[0].sourceId).toBe("OL24333986M"); expect(result).toMatchObject({ - title: "Harry Potter et le prince de sang-mêlé", + title: "Harry Potter et le prince de sang mele", author: "J. K. Rowling", isbn: "9782070612383", isbn13: "9782070612383", @@ -143,7 +208,61 @@ describe("MetadataService", () => { database.onModuleDestroy(); }); - it.runIf(canLoadBetterSqlite())("replaces an ambiguous title-only identification when a later provider supplies a described record", async () => { + it.runIf(canLoadBetterSqlite())("backfills a missing ISBN lookup description from a high-confidence title search", async () => { + const database = createDatabase(); + const openLibraryProvider: MetadataProvider = { + id: "openlibrary", + lookup: async () => ({ + title: "Harry Potter et la coupe de feu", + author: "J. K. Rowling", + isbn: "9782070624553", + publisher: "Gallimard", + publishedDate: "2016" + }), + searchByMetadata: async () => [ + { + title: "Harry Potter et la coupe de feu", + author: "J. K. Rowling", + isbn: "9782070619207", + description: "Harry est invité à assister à la Coupe du monde de Quidditch." + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + openLibraryProvider as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Harry Potter et la coupe de feu", + author: "J. K. Rowling", + description: null, + isbn: "9782070624553", + language: null, + publisher: null, + publishedDate: null, + coverPath: "/covers/local.jpg" + }, + "/library/Harry Potter et la coupe de feu.epub", + { remote: true } + ); + + expect(result).toMatchObject({ + isbn: "9782070624553", + description: "Harry est invité à assister à la Coupe du monde de Quidditch.", + coverPath: "/covers/local.jpg" + }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("uses provider priority as the tie-breaker for high-confidence title matches", async () => { const database = createDatabase(); database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'bnf'").run(); const openLibraryProvider: MetadataProvider = { @@ -179,7 +298,9 @@ describe("MetadataService", () => { providerStub("local") as never, openLibraryProvider as never, providerStub("googlebooks") as never, - bnfProvider as never + bnfProvider as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never ); const localMetadata: BookMetadata = { title: "Daredevil", @@ -196,14 +317,583 @@ describe("MetadataService", () => { expect(result).toMatchObject({ title: "Daredevil", - author: "scénario, Roy Thomas, Gary Friedrich", - isbn: "9782809476255", - isbn13: "9782809476255", + author: "Rosemary Carter", + isbn: "9780373105601", + isbn13: "9780373105601", description: "Daredevil affronte l'Homme aux échasses." }); database.onModuleDestroy(); }); + + it.runIf(canLoadBetterSqlite())("re-enriches from stored local hints instead of a previously failed remote ISBN", async () => { + const database = createDatabase(); + const now = database.now(); + const library = database.db + .insert(libraries) + .values({ name: "Comics", path: "/library", enabled: true, createdAt: now, updatedAt: now }) + .returning() + .get(); + const localMetadataJson = JSON.stringify({ + title: "Daredevil", + author: null, + year: null, + isbn: null, + fileTitle: "Daredevil", + raw: { title: "Daredevil", author: null, publishedDate: null, fileName: "Daredevil" } + }); + const book = database.db + .insert(books) + .values({ + libraryId: library.id, + title: "Daredevil", + author: "Rosemary Carter", + description: null, + isbn: "9780373105601", + isbn13: "9780373105601", + identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }), + localMetadataJson, + language: null, + publisher: "Harlequin Books", + publishedDate: "1982", + format: "cbz", + filePath: "/library/Daredevil.cbz", + coverPath: "/covers/daredevil.jpg", + scanStatus: "succeeded", + enrichmentStatus: "failed", + fileSize: 42, + fileMtime: now, + createdAt: now, + updatedAt: now + }) + .returning() + .get(); + const openLibraryLookup = vi.fn<(_: MetadataLookup, __: MetadataProviderConfig) => Promise>(async () => null); + const bnfSearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise>(async () => [ + { + title: "Daredevil", + author: "scénario, Roy Thomas, Gary Friedrich", + isbn: "9782809476255", + description: "Daredevil affronte l'Homme aux échasses.", + publisher: "Panini comics", + publishedDate: "2019" + } + ]); + database.sqlite.prepare("UPDATE metadata_source_config SET enabled = 1 WHERE provider = 'bnf'").run(); + const service = new MetadataService( + database, + providerStub("local") as never, + { id: "openlibrary", lookup: openLibraryLookup, searchByMetadata: async () => [] } as never, + providerStub("googlebooks") as never, + { id: "bnf", lookup: async () => null, searchByMetadata: bnfSearch } as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichBook(book.id); + + expect(openLibraryLookup).not.toHaveBeenCalled(); + expect(bnfSearch.mock.calls[0]?.[0].title).toBe("Daredevil"); + expect(result).toMatchObject({ + author: "Rosemary Carter", + isbn: "9780373105601", + isbn13: "9780373105601", + publisher: "Harlequin Books", + coverPath: "/covers/daredevil.jpg" + }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("stores provider covers as local bytes and exposes field provenance with metadata status", async () => { + const database = createDatabase(); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + headers: new Headers({ "content-type": "image/jpeg" }), + arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer + } as Response); + const openLibraryProvider: MetadataProvider = { + id: "openlibrary", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Daredevil", + author: "Roy Thomas", + description: "Daredevil affronte une nouvelle menace.", + isbn: "9782809476255", + coverUrl: "https://covers.openlibrary.org/b/id/123-L.jpg" + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + openLibraryProvider as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Daredevil", + author: null, + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Daredevil.cbz", + { remote: true } + ); + + expect(fetchMock).toHaveBeenCalledWith("https://covers.openlibrary.org/b/id/123-L.jpg", expect.any(Object)); + expect(result.coverPath).toMatch(/storage\/covers\/.+\.jpg$/); + expect(result.metadataStatus).toBe("enriched"); + expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({ + title: "local", + author: "openlibrary", + description: "openlibrary", + coverPath: "openlibrary" + }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("retrofits a local cover for an existing book during metadata enrichment", async () => { + const database = createDatabase(); + mkdirSync(database.config.storageDir, { recursive: true }); + const filePath = join(database.config.storageDir, "Demon.Slayer.School.Days.T01.FRENCH.CBZ"); + const zip = new AdmZip(); + zip.addFile("001.jpg", Buffer.from([0xff, 0xd8, 0xff, 0xd9])); + zip.writeZip(filePath); + const now = database.now(); + const library = database.db + .insert(libraries) + .values({ name: "Comics", path: database.config.storageDir, enabled: true, createdAt: now, updatedAt: now }) + .returning() + .get(); + const book = database.db + .insert(books) + .values({ + libraryId: library.id, + title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", + author: null, + description: null, + isbn: null, + isbn13: null, + identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }), + localMetadataJson: JSON.stringify({ + title: "Demon Slayer School Days T01 FRENCH", + author: null, + year: null, + isbn: null, + fileTitle: "Demon Slayer School Days T01 FRENCH", + raw: { title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", author: null, publishedDate: null, fileName: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz" } + }), + language: null, + publisher: null, + publishedDate: null, + format: "cbz", + filePath, + coverPath: null, + metadataStatus: "none", + metadataProvenanceJson: JSON.stringify({ title: "local" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 42, + fileMtime: now, + createdAt: now, + updatedAt: now + }) + .returning() + .get(); + const service = new MetadataService( + database, + providerStub("local") as never, + providerStub("openlibrary") as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichBook(book.id); + + expect(result.coverPath).toMatch(/covers\/[a-f0-9]+\.jpg$/); + expect(result.coverPath && existsSync(result.coverPath)).toBe(true); + expect(result.metadataStatus).toBe("partial"); + expect(JSON.parse(result.metadataProvenanceJson ?? "{}")).toMatchObject({ coverPath: "local" }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("drops sentinel publication dates from provider matches for real affected titles", async () => { + const database = createDatabase(); + const openLibraryProvider: MetadataProvider = { + id: "openlibrary", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Harry Potter et les reliques de la mort", + author: "J. K. Rowling", + publishedDate: "0101-01-01T00:00:00+00:00", + description: "Septième année." + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + openLibraryProvider as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Harry Potter et les reliques de la mort", + author: "J. K. Rowling", + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Harry Potter et les reliques de la mort.epub", + { remote: true } + ); + + expect(result.publishedDate).toBeNull(); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("does not overwrite an existing valid date with a provider sentinel", async () => { + const database = createDatabase(); + const now = database.now(); + const library = database.db + .insert(libraries) + .values({ name: "Novels", path: "/library", enabled: true, createdAt: now, updatedAt: now }) + .returning() + .get(); + const book = database.db + .insert(books) + .values({ + libraryId: library.id, + title: "Lord of the Mysteries", + author: "Cuttlefish That Loves Diving", + description: null, + isbn: null, + isbn13: null, + identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }), + localMetadataJson: JSON.stringify({ + title: "Lord of the Mysteries", + author: "Cuttlefish That Loves Diving", + year: null, + isbn: null, + fileTitle: "Lord of the Mysteries", + raw: { title: "Lord of the Mysteries", author: "Cuttlefish That Loves Diving", publishedDate: null, fileName: "Lord of the Mysteries" } + }), + language: null, + publisher: null, + publishedDate: "2018", + format: "epub", + filePath: "/library/Lord of the Mysteries.epub", + coverPath: null, + metadataStatus: "partial", + metadataProvenanceJson: JSON.stringify({ publishedDate: "existing" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 42, + fileMtime: now, + createdAt: now, + updatedAt: now + }) + .returning() + .get(); + const openLibraryProvider: MetadataProvider = { + id: "openlibrary", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Lord of the Mysteries", + author: "Cuttlefish That Loves Diving", + publishedDate: "0101-01-01T00:00:00+00:00", + description: "A mysterious sequence begins." + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + openLibraryProvider as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + providerStub("mangadex") as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichBook(book.id); + + expect(result.publishedDate).toBe("2018"); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("records MangaDex provenance and stores its cover locally", async () => { + const database = createDatabase(); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + headers: new Headers({ "content-type": "image/jpeg" }), + arrayBuffer: async () => new Uint8Array([9, 8, 7]).buffer + } as Response); + const mangaDexProvider: MetadataProvider = { + id: "mangadex", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Solo Leveling", + author: "Chugong", + description: "A hunter levels up alone.", + publishedDate: "2018", + coverUrl: "https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg" + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + providerStub("openlibrary") as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + mangaDexProvider as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Solo Leveling T03", + author: null, + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Solo Leveling T03.cbz", + { remote: true } + ); + + expect(fetchMock).toHaveBeenCalledWith("https://uploads.mangadex.org/covers/manga-1/cover.jpg.512.jpg", expect.any(Object)); + expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({ + author: "mangadex", + description: "mangadex", + publishedDate: "mangadex", + coverPath: "mangadex" + }); + expect(result.coverPath).toMatch(/storage\/covers\/.+\.jpg$/); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("scores MangaDex matches against the cleaned series title instead of the noisy archive title", async () => { + const database = createDatabase(); + const mangaDexProvider: MetadataProvider = { + id: "mangadex", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Dragon Ball SD", + author: "Naho Ooishi", + description: "A super-deformed Dragon Ball spin-off.", + publishedDate: "2010", + sourceId: "dragon-ball-sd" + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + providerStub("openlibrary") as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + mangaDexProvider as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+", + author: null, + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Dragon.Ball.SD.T01.FRENCH.CBZ.eBook-Paprika+.cbz", + { remote: true } + ); + + expect(result).toMatchObject({ + title: "Dragon Ball SD", + author: "Naho Ooishi", + description: "A super-deformed Dragon Ball spin-off.", + publishedDate: "2010" + }); + expect(JSON.parse(result.metadataProvenanceJson)).toMatchObject({ + title: "local", + author: "mangadex", + description: "mangadex" + }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("accepts MangaDex alias matches through the provider score title", async () => { + const database = createDatabase(); + const mangaDexProvider: MetadataProvider = { + id: "mangadex", + lookup: async () => null, + searchByMetadata: async () => [ + { + title: "Demon Slayer: Kimetsu Academy", + scoreTitle: "Demon Slayer Kimetsu Academy", + author: "Natsuki Hokami", + description: "School spin-off.", + publishedDate: "2021", + sourceId: "kimetsu-academy" + } + ] + }; + const service = new MetadataService( + database, + providerStub("local") as never, + providerStub("openlibrary") as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + mangaDexProvider as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichMetadata( + { + title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", + author: null, + description: null, + isbn: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null + }, + "/library/Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz.cbz", + { remote: true } + ); + + expect(result).toMatchObject({ + title: "Demon Slayer School Days", + author: "Natsuki Hokami", + description: "School spin-off.", + publishedDate: "2021" + }); + + database.onModuleDestroy(); + }); + + it.runIf(canLoadBetterSqlite())("re-enriches existing manga with the cleaned series title on the live book path", async () => { + const database = createDatabase(); + mkdirSync(database.config.storageDir, { recursive: true }); + const filePath = join(database.config.storageDir, "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz.cbz"); + const zip = new AdmZip(); + zip.addFile("001.jpg", Buffer.from([0xff, 0xd8, 0xff, 0xd9])); + zip.writeZip(filePath); + const now = database.now(); + const library = database.db + .insert(libraries) + .values({ name: "Manga", path: database.config.storageDir, enabled: true, createdAt: now, updatedAt: now }) + .returning() + .get(); + const book = database.db + .insert(books) + .values({ + libraryId: library.id, + title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", + author: null, + description: null, + isbn: null, + isbn13: null, + identifiersJson: JSON.stringify({ isbn10: null, isbn13: null, candidates: [] }), + localMetadataJson: JSON.stringify({ + title: "Demon Slayer School Days T01 FRENCH", + author: null, + year: null, + isbn: null, + fileTitle: "Demon Slayer School Days T01 FRENCH", + raw: { + title: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz", + author: null, + publishedDate: null, + fileName: "Demon.Slayer.School.Days.T01.FRENCH.CBZ. -ebdz" + } + }), + language: null, + publisher: null, + publishedDate: null, + format: "cbz", + filePath, + coverPath: null, + metadataStatus: "none", + metadataProvenanceJson: JSON.stringify({ title: "local" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 42, + fileMtime: now, + createdAt: now, + updatedAt: now + }) + .returning() + .get(); + const mangaDexSearch = vi.fn<(_: MetadataSearchQuery, __: MetadataProviderConfig) => Promise>(async () => [ + { + title: "Demon Slayer School Days", + author: "Natsuki Hokami", + description: "School spin-off.", + publishedDate: "2021", + sourceId: "demon-slayer-school-days" + } + ]); + const service = new MetadataService( + database, + providerStub("local") as never, + providerStub("openlibrary") as never, + providerStub("googlebooks") as never, + providerStub("bnf") as never, + { id: "mangadex", lookup: async () => null, searchByMetadata: mangaDexSearch } as never, + providerStub("comicvine") as never + ); + + const result = await service.enrichBook(book.id); + + expect(mangaDexSearch.mock.calls[0]?.[0].title).toBe("Demon Slayer School Days"); + expect(result).toMatchObject({ + title: "Demon Slayer School Days", + author: "Natsuki Hokami", + description: "School spin-off.", + publishedDate: "2021" + }); + + database.onModuleDestroy(); + }); }); function createDatabase(): DatabaseService { diff --git a/apps/api/src/metadata/metadata.service.ts b/apps/api/src/metadata/metadata.service.ts index 1553ba0..005ca2a 100644 --- a/apps/api/src/metadata/metadata.service.ts +++ b/apps/api/src/metadata/metadata.service.ts @@ -1,21 +1,45 @@ import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, extname, join } from "node:path"; import { eq } from "drizzle-orm"; import { MetadataSourcesConfigDto, UpdateMetadataSourcesConfigDto } from "@readabook/shared"; import { DatabaseService } from "../database/database.service.js"; -import { automationSettings, books, metadataSourceConfig } from "../database/schema.js"; -import { BookMetadata } from "../scanner/metadata.js"; +import { automationSettings, books, metadataSourceConfig, series } from "../database/schema.js"; +import { BookMetadata, extractMetadata } from "../scanner/metadata.js"; import { BnfProvider } from "./adapters/bnf.provider.js"; +import { ComicVineProvider } from "./adapters/comic-vine.provider.js"; import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; import { LocalMetadataProvider } from "./adapters/local.provider.js"; +import { MangaDexProvider } from "./adapters/mangadex.provider.js"; import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; -import { BookIdentifiers, LocalMetadataHints, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js"; +import { describeMetadataProviderError, providerFetch } from "./adapters/provider-fetch.js"; +import { + BookIdentifiers, + LocalMetadataHints, + MetadataField, + MetadataMatch, + MetadataProvider, + MetadataProviderConfig, + MetadataProviderId, + MetadataProvenance, + MetadataSearchQuery, + MetadataStatus +} from "./metadata.types.js"; import { ExtractIdentifiers, toIsbn13 } from "./use-cases/extract-identifiers.js"; import { ExtractLocalMetadataHints } from "./use-cases/extract-local-metadata-hints.js"; +import { extractSeriesVolume } from "./use-cases/extract-series-volume.js"; +import { normalizePublishedDate } from "./use-cases/normalize-published-date.js"; import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js"; -import { ScoreMetadataMatch } from "./use-cases/score-metadata-match.js"; +import { ScoredMetadataMatch, ScoreMetadataMatch } from "./use-cases/score-metadata-match.js"; + +type ProviderCandidate = ScoredMetadataMatch & { + provider: MetadataProviderId; + priority: number; +}; @Injectable() export class MetadataService { @@ -29,9 +53,11 @@ export class MetadataService { local: LocalMetadataProvider, openLibrary: OpenLibraryProvider, googleBooks: GoogleBooksProvider, - bnf: BnfProvider + bnf: BnfProvider, + mangaDex: MangaDexProvider, + comicVine: ComicVineProvider ) { - this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf]); + this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf, mangaDex, comicVine]); } getSourcesConfig(): MetadataSourcesConfigDto { @@ -75,96 +101,119 @@ export class MetadataService { localMetadata: BookMetadata, filePath: string, options: { remote: boolean } - ): Promise { + ): Promise< + BookMetadata & { + isbn13: string | null; + identifiersJson: string; + localMetadataJson: string; + metadataStatus: MetadataStatus; + metadataProvenanceJson: string; + } + > { const identifiers = this.extractIdentifiers.fromMetadataAndFile(localMetadata, filePath); const local = this.extractLocalMetadataHints.fromMetadataAndFile(localMetadata, filePath); const configs = this.getProviderConfigs(); const chain = options.remote ? this.resolveProviderChain.resolve(configs) : this.resolveProviderChain.resolve(configs).filter((entry) => entry.provider.id === "local"); - let merged: BookMetadata = { ...localMetadata }; + const candidates: ProviderCandidate[] = []; + const query = this.buildSearchQuery(local, identifiers, filePath); for (const { provider, config } of chain) { + if (provider.id === "local") continue; try { const hasIsbn = Boolean(identifiers.isbn13 ?? identifiers.isbn10); - const match = - provider.id === "local" || hasIsbn - ? await provider.lookup( - { - title: merged.title, - author: merged.author, - filePath, - sourceId: null, - identifiers, - local - }, - config - ) - : null; + const match = hasIsbn + ? await provider.lookup( + { + title: local.title, + author: local.author, + filePath, + sourceId: null, + identifiers, + local + }, + config + ) + : null; if (match) { - merged = mergeMetadata(merged, match); + const completedMatch = match.description + ? match + : mergeMetadataMatch(match, await this.searchMissingDescription(provider, config, local, identifiers, filePath)); + candidates.push(scoreProviderCandidate(this.scoreMetadataMatch, query, completedMatch, provider.id, config.priority)); continue; } - if (!options.remote || provider.id === "local") continue; - const query = { - title: local.title, - author: local.author, - year: local.year, - isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn - }; - const best = this.scoreMetadataMatch.best(query, await provider.searchByMetadata(query, config)); + if (!options.remote) continue; + const matches = await provider.searchByMetadata(query, config); + const best = this.scoreMetadataMatch.best(query, matches); + if (!best && matches.length) { + console.info( + `[metadata] Provider ${provider.id} returned ${matches.length} result(s) rejected by scoring for "${query.title}"` + ); + } if (best) { if (!isActionableSearchMatch(best.match)) continue; - merged = shouldReplaceAmbiguousIdentification(localMetadata, merged, best.match) - ? mergeMetadata({ ...localMetadata, author: null, isbn: null, description: null, language: null, publisher: null, publishedDate: null }, best.match) - : mergeMetadata(merged, best.match); + let providerMatch = best.match; - const detailedMatch = await this.lookupSearchMatchDetails(provider, config, filePath, identifiers, local, merged, best.match); - if (detailedMatch) merged = mergeMetadata(merged, detailedMatch); + const detailedMatch = await this.lookupSearchMatchDetails(provider, config, filePath, identifiers, local, best.match); + if (detailedMatch) providerMatch = mergeMetadataMatch(detailedMatch, best.match); + candidates.push(scoreProviderCandidate(this.scoreMetadataMatch, query, providerMatch, provider.id, config.priority)); } - } catch { - // Provider failures must not block local ingestion. + } catch (error) { + console.warn(`[metadata] Provider ${provider.id} failed for "${query.title}": ${describeMetadataProviderError(error)}`); } } + const materializedCandidates = await this.materializeQualifiedCovers(candidates, filePath); + const localProvenance = provenanceFromLocal(localMetadata); + const { metadata: merged, provenance: remoteProvenance } = mergeCandidatesWithLocal( + materializedCandidates, + localMetadata, + identifiers, + query.title + ); + let provenance: MetadataProvenance = { ...localProvenance, ...remoteProvenance }; + if (merged.isbn && !provenance.isbn) provenance.isbn = "local"; const isbn13 = identifiers.isbn13 ?? (merged.isbn ? toIsbn13(merged.isbn) : null); + const metadataStatus = computeMetadataStatus(merged); return { ...merged, isbn: merged.isbn ?? isbn13 ?? identifiers.isbn10, isbn13, identifiersJson: JSON.stringify(identifiers), - localMetadataJson: JSON.stringify(local) + localMetadataJson: JSON.stringify(local), + metadataStatus, + metadataProvenanceJson: JSON.stringify(provenance) }; } async enrichBook(bookId: number): Promise { const book = this.database.db.select().from(books).where(eq(books.id, bookId)).get(); if (!book) throw new NotFoundException("Book not found"); - const metadata: BookMetadata = { - title: book.title, - author: book.author, - description: book.description, - isbn: book.isbn, - language: book.language, - publisher: book.publisher, - publishedDate: book.publishedDate, - coverPath: book.coverPath - }; + const local = parseStoredLocalMetadata(book.localMetadataJson); + const metadata = await this.extractCurrentLocalMetadata(book, local); const enriched = await this.enrichMetadata(metadata, book.filePath, { remote: true }); + const next = preserveExistingWhenMissing(enriched, book, book.filePath); + const seriesInfo = this.resolveSeries(next.title, book.filePath); return this.database.db .update(books) .set({ - title: enriched.title, - author: enriched.author, - description: enriched.description, - isbn: enriched.isbn, - isbn13: enriched.isbn13, + seriesId: seriesInfo.seriesId, + title: next.title, + author: next.author, + description: next.description, + isbn: next.isbn, + isbn13: next.isbn13, identifiersJson: enriched.identifiersJson, localMetadataJson: enriched.localMetadataJson, - language: enriched.language, - publisher: enriched.publisher, - publishedDate: enriched.publishedDate, - coverPath: enriched.coverPath, + language: next.language, + publisher: next.publisher, + publishedDate: next.publishedDate, + volumeNumber: seriesInfo.volumeNumber, + volumeLabel: seriesInfo.volumeLabel, + coverPath: next.coverPath, + metadataStatus: next.metadataStatus, + metadataProvenanceJson: next.metadataProvenanceJson, updatedAt: this.database.now() }) .where(eq(books.id, book.id)) @@ -172,6 +221,57 @@ export class MetadataService { .get(); } + private async extractCurrentLocalMetadata(book: typeof books.$inferSelect, local: LocalMetadataHints | null): Promise { + const fallback: BookMetadata = { + title: local?.title ?? book.title, + author: local ? local.author : book.author, + description: null, + isbn: local ? local.isbn : book.isbn, + language: null, + publisher: book.publisher, + publishedDate: normalizePublishedDate(local ? local.year : book.publishedDate), + coverPath: book.coverPath + }; + if (!existsSync(book.filePath)) return fallback; + try { + const extracted = await extractMetadata(book.filePath, this.database.config.storageDir); + return { + title: extracted.title || fallback.title, + author: extracted.author ?? fallback.author, + description: extracted.description ?? fallback.description, + isbn: extracted.isbn ?? fallback.isbn, + language: extracted.language ?? fallback.language, + publisher: extracted.publisher ?? fallback.publisher, + publishedDate: normalizePublishedDate(extracted.publishedDate) ?? fallback.publishedDate, + coverPath: extracted.coverPath ?? fallback.coverPath + }; + } catch { + return fallback; + } + } + + private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } { + const parsed = extractSeriesVolume(title, filePath); + const now = this.database.now(); + const row = this.database.db + .insert(series) + .values({ + title: parsed.seriesTitle, + normalizedTitle: parsed.normalizedSeriesTitle, + description: null, + publisher: null, + createdAt: now, + updatedAt: now + }) + .onConflictDoUpdate({ + target: series.normalizedTitle, + set: { title: parsed.seriesTitle, updatedAt: now } + }) + .returning({ id: series.id }) + .get(); + return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel }; + } + private getProviderConfigs(): MetadataProviderConfig[] { return this.database.db .select() @@ -195,7 +295,6 @@ export class MetadataService { filePath: string, identifiers: BookIdentifiers, local: LocalMetadataHints, - merged: BookMetadata, match: MetadataMatch ): Promise { const derivedIdentifiers = { @@ -206,13 +305,12 @@ export class MetadataService { }; const hasNewIdentifier = derivedIdentifiers.isbn13 !== identifiers.isbn13 || derivedIdentifiers.isbn10 !== identifiers.isbn10; const hasLookupTarget = hasNewIdentifier || Boolean(match.sourceId); - const needsDetails = !merged.description && hasLookupTarget; - if (!hasLookupTarget && !needsDetails) return null; + if (!hasLookupTarget) return null; return provider.lookup( { - title: merged.title, - author: merged.author, + title: match.title ?? local.title, + author: match.author ?? local.author, filePath, sourceId: match.sourceId, identifiers: derivedIdentifiers, @@ -221,26 +319,309 @@ export class MetadataService { config ); } + + private async searchMissingDescription( + provider: MetadataProvider, + config: MetadataProviderConfig, + local: LocalMetadataHints, + identifiers: BookIdentifiers, + filePath: string + ): Promise { + const query = this.buildSearchQuery(local, identifiers, filePath); + const matches = await provider.searchByMetadata(query, config); + return ( + matches + .map((match) => ({ match, score: this.scoreMetadataMatch.score(query, match) })) + .filter((entry) => entry.match.description && entry.score >= 75) + .sort((left, right) => right.score - left.score)[0]?.match ?? null + ); + } + + private buildSearchQuery(local: LocalMetadataHints, identifiers: BookIdentifiers, filePath: string): MetadataSearchQuery { + return { + title: extractSeriesVolume(local.title, filePath).seriesTitle, + author: local.author, + year: local.year, + isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn + }; + } + + private async materializeCover(match: MetadataMatch, filePath: string, provider: MetadataProviderId): Promise { + if (match.coverPath || !match.coverUrl) return match; + try { + const response = await providerFetch("cover", match.coverUrl, { timeoutMs: 5000 }); + if (!response.ok) return match; + const data = Buffer.from(await response.arrayBuffer()); + if (!data.length) return match; + const extension = coverExtension(match.coverUrl, response.headers.get("content-type")); + const hash = createHash("sha256").update(`${filePath}:${provider}:${match.coverUrl}`).digest("hex").slice(0, 24); + const target = join(this.database.config.storageDir, "covers", `${hash}${extension}`); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, data); + return { ...match, coverPath: target }; + } catch { + return match; + } + } + + private async materializeQualifiedCovers(candidates: ProviderCandidate[], filePath: string): Promise { + const materialized: ProviderCandidate[] = []; + for (const candidate of candidates) { + if (isQualifiedSourceCover(candidate)) { + materialized.push({ + ...candidate, + match: await this.materializeCover(candidate.match, filePath, candidate.provider) + }); + } else { + materialized.push(candidate); + } + } + return materialized; + } } function isActionableSearchMatch(match: MetadataMatch): boolean { return Boolean(match.isbn ?? match.sourceId ?? match.description); } -function shouldReplaceAmbiguousIdentification(local: BookMetadata, current: BookMetadata, next: MetadataMatch): boolean { - if (local.author || local.isbn || !current.isbn || !next.isbn || current.isbn === next.isbn) return false; - return Boolean(next.description); -} - -function mergeMetadata(current: BookMetadata, next: MetadataMatch): BookMetadata { +function mergeMetadataMatch(current: MetadataMatch, next: MetadataMatch | null): MetadataMatch { + if (!next) return current; return { - title: next.title ?? current.title, - author: current.author ?? next.author ?? null, - description: current.description ?? next.description ?? null, - isbn: current.isbn ?? next.isbn ?? null, - language: current.language ?? next.language ?? null, - publisher: current.publisher ?? next.publisher ?? null, - publishedDate: current.publishedDate ?? next.publishedDate ?? null, - coverPath: current.coverPath ?? next.coverPath ?? null + title: current.title ?? next.title, + author: current.author ?? next.author, + description: current.description ?? next.description, + isbn: current.isbn ?? next.isbn, + language: current.language ?? next.language, + publisher: current.publisher ?? next.publisher, + publishedDate: normalizePublishedDate(current.publishedDate) ?? normalizePublishedDate(next.publishedDate), + coverPath: current.coverPath ?? next.coverPath, + coverUrl: current.coverUrl ?? next.coverUrl, + sourceId: current.sourceId ?? next.sourceId, + identifiers: current.identifiers ?? next.identifiers }; } + +const metadataFields: MetadataField[] = ["title", "author", "description", "isbn", "language", "publisher", "publishedDate", "coverPath"]; +const fillableMetadataFields: MetadataField[] = ["author", "description", "isbn", "language", "publisher", "publishedDate"]; + +function scoreProviderCandidate( + scorer: ScoreMetadataMatch, + query: MetadataSearchQuery, + match: MetadataMatch, + provider: MetadataProviderId, + priority: number +): ProviderCandidate { + return { ...scorer.details(query, match), provider, priority }; +} + +function mergeCandidatesWithLocal( + candidates: ProviderCandidate[], + local: BookMetadata, + identifiers: BookIdentifiers, + title: string +): { metadata: BookMetadata; provenance: MetadataProvenance } { + const sorted = [...candidates].sort(compareProviderCandidates); + const retained = sorted[0] ?? null; + const completionOrder = [...(retained ? [retained] : []), ...sorted.filter((candidate) => candidate !== retained)]; + const metadata: BookMetadata = { + title, + author: local.author ?? null, + description: local.description ?? null, + isbn: identifiers.isbn13 ?? identifiers.isbn10 ?? local.isbn ?? null, + language: local.language ?? null, + publisher: local.publisher ?? null, + publishedDate: normalizePublishedDate(local.publishedDate), + coverPath: local.coverPath ?? null + }; + const provenance: MetadataProvenance = {}; + for (const field of fillableMetadataFields) { + if (hasMetadataValue(metadata[field])) continue; + const source = completionOrder.find((candidate) => hasMetadataValue(normalizeCandidateField(candidate.match, field))); + if (!source) continue; + metadata[field] = normalizeCandidateField(source.match, field) as never; + provenance[field] = source.provider; + provenance[`${field}Score` as MetadataField] = String(source.score) as never; + } + const coverSource = completionOrder.find((candidate) => isQualifiedSourceCover(candidate) && hasMetadataValue(candidate.match.coverPath)); + if (coverSource && shouldUseSourceCover(metadata.coverPath)) { + metadata.coverPath = coverSource.match.coverPath ?? null; + provenance.coverPath = coverSource.provider; + provenance.coverPathScore = String(coverSource.score) as never; + } + return { metadata, provenance }; +} + +function compareProviderCandidates(left: ProviderCandidate, right: ProviderCandidate): number { + if (left.isbnMatch !== right.isbnMatch) return left.isbnMatch ? -1 : 1; + const leftHighConfidence = isHighConfidenceSelection(left); + const rightHighConfidence = isHighConfidenceSelection(right); + if (leftHighConfidence && rightHighConfidence) return left.priority - right.priority; + if (leftHighConfidence !== rightHighConfidence) return leftHighConfidence ? -1 : 1; + if (left.score !== right.score) return right.score - left.score; + return left.priority - right.priority; +} + +function isHighConfidenceSelection(candidate: ProviderCandidate): boolean { + return candidate.titleScore > 90 && (candidate.authorScore == null || candidate.authorScore >= 15); +} + +function isQualifiedSourceCover(candidate: ProviderCandidate): boolean { + return candidate.score >= 80 && candidate.titleScore >= 85 && Boolean(candidate.match.coverPath ?? candidate.match.coverUrl); +} + +function shouldUseSourceCover(currentCoverPath: string | null): boolean { + return !hasMetadataValue(currentCoverPath) || isLocalCoverPath(currentCoverPath); +} + +function isLocalCoverPath(value: string): boolean { + return /[/\\]covers[/\\][a-f0-9]{24}\.[a-z0-9]+$/i.test(value); +} + +function normalizeCandidateField(match: MetadataMatch, field: MetadataField): string | null { + if (field === "publishedDate") return normalizePublishedDate(match.publishedDate); + return match[field] ?? null; +} + +function hasMetadataValue(value: string | null | undefined): value is string { + return Boolean(value && value.trim()); +} + +function provenanceFromLocal(local: BookMetadata): MetadataProvenance { + const provenance: MetadataProvenance = {}; + for (const field of metadataFields) { + if (local[field] != null && local[field] !== "") provenance[field] = "local"; + } + return provenance; +} + +function computeMetadataStatus(metadata: Pick): MetadataStatus { + const hasCover = Boolean(metadata.coverPath); + const filled = [metadata.author, metadata.description, metadata.isbn, metadata.language, metadata.publisher, metadata.publishedDate].filter(Boolean).length; + if (hasCover && filled >= 2) return "enriched"; + if (hasCover || filled > 0) return "partial"; + return "none"; +} + +function mergeExistingProvenance( + enriched: BookMetadata & { metadataProvenanceJson: string }, + existing: typeof books.$inferSelect, + finalValues: BookMetadata & { isbn13: string | null } +): MetadataProvenance { + const next = parseProvenance(enriched.metadataProvenanceJson); + const previous = parseProvenance(existing.metadataProvenanceJson); + const provenance: MetadataProvenance = { ...previous }; + for (const field of metadataFields) { + if (field === "title") { + if (finalValues.title && !provenance.title) { + provenance.title = finalValues.title === enriched.title && finalValues.title !== existing.title ? (next.title ?? "local") : (previous.title ?? "existing"); + } + continue; + } + if (field === "publishedDate") { + const finalDate = normalizePublishedDate(finalValues.publishedDate); + if (finalDate && finalDate === normalizePublishedDate(enriched.publishedDate) && finalDate !== normalizePublishedDate(existing.publishedDate)) { + provenance[field] = next[field] ?? provenance[field]; + copyScoreProvenance(next, provenance, field); + } else if (finalDate && !provenance[field]) { + provenance[field] = previous[field] ?? "existing"; + } + continue; + } + if (field === "coverPath" && finalValues.coverPath && finalValues.coverPath === enriched.coverPath && finalValues.coverPath !== existing.coverPath) { + provenance.coverPath = next.coverPath ?? provenance.coverPath; + copyScoreProvenance(next, provenance, field); + continue; + } + if (finalValues[field] && finalValues[field] === enriched[field] && finalValues[field] !== existing[field]) { + provenance[field] = next[field] ?? provenance[field]; + copyScoreProvenance(next, provenance, field); + continue; + } + if (finalValues[field] != null && existing[field] != null && !provenance[field]) { + provenance[field] = previous[field] ?? "existing"; + } + } + return provenance; +} + +function copyScoreProvenance(source: MetadataProvenance, target: MetadataProvenance, field: MetadataField): void { + const scoreKey = `${field}Score`; + if (source[scoreKey]) target[scoreKey] = source[scoreKey]; +} + +function parseProvenance(value: string | null): MetadataProvenance { + if (!value) return {}; + try { + const parsed = JSON.parse(value) as MetadataProvenance; + return parsed && typeof parsed === "object" ? parsed : {}; + } catch { + return {}; + } +} + +function coverExtension(url: string, contentType: string | null): string { + if (contentType?.includes("png")) return ".png"; + if (contentType?.includes("webp")) return ".webp"; + if (contentType?.includes("gif")) return ".gif"; + const fromUrl = extname(new URL(url).pathname).toLowerCase(); + return fromUrl === ".png" || fromUrl === ".webp" || fromUrl === ".gif" || fromUrl === ".jpg" || fromUrl === ".jpeg" ? fromUrl : ".jpg"; +} + +function parseStoredLocalMetadata(value: string | null): LocalMetadataHints | null { + if (!value) return null; + try { + const parsed = JSON.parse(value) as Partial; + return typeof parsed.title === "string" ? (parsed as LocalMetadataHints) : null; + } catch { + return null; + } +} + +function preserveExistingWhenMissing( + enriched: BookMetadata & { + isbn13: string | null; + identifiersJson: string; + localMetadataJson: string; + metadataStatus: MetadataStatus; + metadataProvenanceJson: string; + }, + existing: typeof books.$inferSelect, + filePath: string +): BookMetadata & { isbn13: string | null; metadataStatus: MetadataStatus; metadataProvenanceJson: string } { + const previousProvenance = parseProvenance(existing.metadataProvenanceJson); + const next = { + title: chooseTitle(existing.title, enriched.title, filePath), + author: existing.author ?? enriched.author, + description: existing.description ?? enriched.description, + isbn: existing.isbn ?? enriched.isbn, + isbn13: existing.isbn13 ?? enriched.isbn13, + language: existing.language ?? enriched.language, + publisher: existing.publisher ?? enriched.publisher, + publishedDate: normalizePublishedDate(existing.publishedDate) ?? normalizePublishedDate(enriched.publishedDate), + coverPath: chooseCoverPath(existing.coverPath, enriched.coverPath, previousProvenance) + }; + const provenance = mergeExistingProvenance(enriched, existing, next); + return { + ...next, + metadataStatus: computeMetadataStatus(next), + metadataProvenanceJson: JSON.stringify(provenance) + }; +} + +function chooseCoverPath(existingCoverPath: string | null, enrichedCoverPath: string | null, previousProvenance: MetadataProvenance): string | null { + if (!existingCoverPath) return enrichedCoverPath; + if (!enrichedCoverPath || enrichedCoverPath === existingCoverPath) return existingCoverPath; + return canReplaceExistingCover(existingCoverPath, previousProvenance) ? enrichedCoverPath : existingCoverPath; +} + +function chooseTitle(existingTitle: string, enrichedTitle: string, filePath: string): string { + if (!enrichedTitle) return existingTitle; + if (!existingTitle) return enrichedTitle; + const parsedExisting = extractSeriesVolume(existingTitle, filePath).seriesTitle; + return parsedExisting === enrichedTitle && existingTitle !== enrichedTitle ? enrichedTitle : existingTitle; +} + +function canReplaceExistingCover(existingCoverPath: string, previousProvenance: MetadataProvenance): boolean { + const provenance = previousProvenance.coverPath; + return (provenance == null || provenance === "local" || provenance === "existing") && isLocalCoverPath(existingCoverPath); +} diff --git a/apps/api/src/metadata/metadata.types.ts b/apps/api/src/metadata/metadata.types.ts index c23b836..f2200da 100644 --- a/apps/api/src/metadata/metadata.types.ts +++ b/apps/api/src/metadata/metadata.types.ts @@ -1,6 +1,6 @@ import { BookMetadata } from "../scanner/metadata.js"; -export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf"; +export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf" | "mangadex" | "comicvine"; export type BookIdentifiers = { isbn10: string | null; @@ -40,9 +40,17 @@ export type MetadataSearchQuery = { export type MetadataMatch = Partial & { sourceId?: string | null; + coverUrl?: string | null; identifiers?: Partial; + scoreTitle?: string | null; }; +export type MetadataField = "title" | "author" | "description" | "isbn" | "language" | "publisher" | "publishedDate" | "coverPath"; + +export type MetadataStatus = "enriched" | "partial" | "none"; + +export type MetadataProvenance = Partial>; + export type MetadataProviderConfig = { provider: MetadataProviderId; enabled: boolean; diff --git a/apps/api/src/metadata/normalize-published-date.test.ts b/apps/api/src/metadata/normalize-published-date.test.ts new file mode 100644 index 0000000..5bd62f0 --- /dev/null +++ b/apps/api/src/metadata/normalize-published-date.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { normalizePublishedDate } from "./use-cases/normalize-published-date.js"; + +describe("normalizePublishedDate", () => { + it("rejects sentinel and absurd dates seen in real metadata providers", () => { + expect(normalizePublishedDate("0101-01-01T00:00:00+00:00")).toBeNull(); + expect(normalizePublishedDate("0001-01-01")).toBeNull(); + expect(normalizePublishedDate("1970-01-01")).toBeNull(); + expect(normalizePublishedDate("0000")).toBeNull(); + }); + + it("keeps only credible supported date formats", () => { + expect(normalizePublishedDate("2007")).toBe("2007"); + expect(normalizePublishedDate("2007-07")).toBe("2007-07"); + expect(normalizePublishedDate("2007-07-21")).toBe("2007-07-21"); + expect(normalizePublishedDate("2007-07-21T00:00:00+00:00")).toBe("2007-07-21"); + }); + + it("rejects years outside the supported publication range", () => { + expect(normalizePublishedDate("1499")).toBeNull(); + expect(normalizePublishedDate("2028")).toBeNull(); + }); +}); diff --git a/apps/api/src/metadata/use-cases/normalize-published-date.ts b/apps/api/src/metadata/use-cases/normalize-published-date.ts new file mode 100644 index 0000000..9c7fe72 --- /dev/null +++ b/apps/api/src/metadata/use-cases/normalize-published-date.ts @@ -0,0 +1,44 @@ +const minimumYear = 1500; +const maximumYear = 2027; +const rejectedExactDates = new Set(["0001-01-01", "0101-01-01", "1970-01-01"]); + +export function normalizePublishedDate(value: string | null | undefined): string | null { + const text = value?.trim(); + if (!text) return null; + + const isoDate = text.match(/^(\d{4})-(\d{2})-(\d{2})(?:[T\s].*)?$/); + if (isoDate) { + const [, year, month, day] = isoDate; + const date = `${year}-${month}-${day}`; + if (rejectedExactDates.has(date)) return null; + return validDate(Number(year), Number(month), Number(day)) ? date : null; + } + + const yearMonth = text.match(/^(\d{4})-(\d{2})$/); + if (yearMonth) { + const [, year, month] = yearMonth; + return validYear(Number(year)) && validMonth(Number(month)) ? `${year}-${month}` : null; + } + + const yearOnly = text.match(/^(\d{4})$/); + if (yearOnly) { + const year = Number(yearOnly[1]); + return validYear(year) ? yearOnly[1] : null; + } + + return null; +} + +function validDate(year: number, month: number, day: number): boolean { + if (!validYear(year) || !validMonth(month) || day < 1 || day > 31) return false; + const date = new Date(Date.UTC(year, month - 1, day)); + return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day; +} + +function validYear(year: number): boolean { + return Number.isInteger(year) && year >= minimumYear && year <= maximumYear; +} + +function validMonth(month: number): boolean { + return Number.isInteger(month) && month >= 1 && month <= 12; +} diff --git a/apps/api/src/metadata/use-cases/score-metadata-match.ts b/apps/api/src/metadata/use-cases/score-metadata-match.ts index a81de00..84e16b3 100644 --- a/apps/api/src/metadata/use-cases/score-metadata-match.ts +++ b/apps/api/src/metadata/use-cases/score-metadata-match.ts @@ -3,56 +3,125 @@ import { MetadataMatch, MetadataSearchQuery } from "../metadata.types.js"; export type ScoredMetadataMatch = { match: MetadataMatch; score: number; + titleScore: number; + authorScore: number | null; + dateScore: number | null; + isbnMatch: boolean; }; export class ScoreMetadataMatch { score(query: MetadataSearchQuery, match: MetadataMatch): number { - let score = 0; - const titleScore = similarity(normalize(query.title), normalize(match.title ?? "")); - score += titleScore * 0.7; - - if (query.author && match.author) { - score += similarity(normalize(query.author), normalize(match.author)) * 0.2; - } else if (!query.author) { - score += 0.08; - } - - const queryYear = query.year ?? null; - const matchYear = match.publishedDate?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null; - if (queryYear && matchYear) score += queryYear === matchYear ? 0.1 : -0.1; - - if (query.isbn && match.isbn && query.isbn.replace(/\D/g, "") === match.isbn.replace(/\D/g, "")) { - score += 0.25; - } - return Math.max(0, Math.min(1, score)); + return this.details(query, match).score; } - best(query: MetadataSearchQuery, matches: MetadataMatch[], minimumScore = 0.55): ScoredMetadataMatch | null { + details(query: MetadataSearchQuery, match: MetadataMatch): ScoredMetadataMatch { + const titleScore = scoreTitle(query.title, match.scoreTitle ?? match.title ?? ""); + const authorScore = query.author ? scoreAuthor(query.author, match.author) : null; + const dateScore = query.year ? scoreDate(query.year, match.publishedDate) : null; + const isbnMatch = exactIsbnMatch(query.isbn, match.isbn); + const maxPossible = 100 + (authorScore == null ? 0 : 30) + (dateScore == null ? 0 : 10); + const sum = titleScore + (authorScore ?? 0) + (dateScore ?? 0); + return { + match, + score: maxPossible ? Math.round((100 * sum) / maxPossible) : 0, + titleScore, + authorScore, + dateScore, + isbnMatch + }; + } + + best(query: MetadataSearchQuery, matches: MetadataMatch[], minimumScore = 0): ScoredMetadataMatch | null { const scored = matches - .map((match) => ({ match, score: this.score(query, match) })) - .sort((left, right) => right.score - left.score); + .map((match) => this.details(query, match)) + .sort((left, right) => compareScoredMatches(query, left, right)); const best = scored[0]; return best && best.score >= minimumScore ? best : null; } } -function normalize(value: string): string { +function compareScoredMatches(query: MetadataSearchQuery, left: ScoredMetadataMatch, right: ScoredMetadataMatch): number { + if (left.isbnMatch !== right.isbnMatch) return left.isbnMatch ? -1 : 1; + const leftTitleAuthor = isHighConfidenceTitleAuthor(query, left); + const rightTitleAuthor = isHighConfidenceTitleAuthor(query, right); + if (leftTitleAuthor !== rightTitleAuthor) return leftTitleAuthor ? -1 : 1; + return right.score - left.score; +} + +function isHighConfidenceTitleAuthor(query: MetadataSearchQuery, scored: ScoredMetadataMatch): boolean { + return scored.titleScore > 90 && (!query.author || (scored.authorScore ?? 0) >= 15); +} + +function scoreTitle(left: string, right: string): number { + const normalizedLeft = normalizeTitle(left); + const normalizedRight = normalizeTitle(right); + if (!normalizedLeft || !normalizedRight) return 0; + if (normalizedLeft === normalizedRight) return 100; + if (normalizedLeft.includes(normalizedRight) || normalizedRight.includes(normalizedLeft)) return 95; + return Math.round(jaccard(tokens(normalizedLeft), tokens(normalizedRight)) * 100); +} + +function scoreAuthor(localAuthor: string, sourceAuthor: string | null | undefined): number { + const local = authorSet(localAuthor); + if (!local.size) return 0; + const source = authorSet(sourceAuthor ?? ""); + const present = [...local].filter((author) => source.has(author)).length; + return 30 * (present / local.size); +} + +function scoreDate(localYear: string, sourceDate: string | null | undefined): number { + const left = Number(yearFrom(localYear)); + const right = Number(yearFrom(sourceDate ?? "")); + if (!left || !right) return 0; + if (left === right) return 10; + return Math.abs(left - right) <= 1 ? 5 : 0; +} + +function exactIsbnMatch(left: string | null | undefined, right: string | null | undefined): boolean { + const normalizedLeft = normalizeIsbn(left); + const normalizedRight = normalizeIsbn(right); + return Boolean(normalizedLeft && normalizedRight && normalizedLeft === normalizedRight); +} + +function normalizeTitle(value: string): string { + return normalizeText(value.split(":")[0] ?? "").replace(/^(?:le|la|les|the|a|an|l)\s+/, ""); +} + +function normalizeText(value: string): string { return value .normalize("NFD") .replace(/[\u0300-\u036f]/g, "") .toLowerCase() .replace(/[^a-z0-9]+/g, " ") - .replace(/\b(le|la|les|the|a|an|de|du|des|et|and)\b/g, " ") .replace(/\s+/g, " ") .trim(); } -function similarity(left: string, right: string): number { - if (!left || !right) return 0; - if (left === right) return 1; - const leftTokens = new Set(left.split(" ")); - const rightTokens = new Set(right.split(" ")); +function authorSet(value: string): Set { + return new Set( + value + .split(/[,;&/]|\band\b|\bet\b/gi) + .map(normalizeText) + .filter(Boolean) + .sort() + ); +} + +function tokens(value: string): Set { + return new Set(value.split(" ").filter(Boolean)); +} + +function jaccard(leftTokens: Set, rightTokens: Set): number { const intersection = [...leftTokens].filter((token) => rightTokens.has(token)).length; const union = new Set([...leftTokens, ...rightTokens]).size; return union ? intersection / union : 0; } + +function yearFrom(value: string): string | null { + return value.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null; +} + +function normalizeIsbn(value: string | null | undefined): string | null { + const normalized = value?.replace(/[^0-9X]/gi, "").toUpperCase() ?? ""; + return normalized || null; +} diff --git a/apps/api/src/scanner/metadata.test.ts b/apps/api/src/scanner/metadata.test.ts index 18e0b9a..c7ca2b4 100644 --- a/apps/api/src/scanner/metadata.test.ts +++ b/apps/api/src/scanner/metadata.test.ts @@ -2,10 +2,15 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import AdmZip from "adm-zip"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { listCbzImageEntries } from "../common/cbz.js"; import { extractMetadata } from "./metadata.js"; +vi.mock("../common/cbr.js", () => ({ + listCbrImageEntries: async () => [{ entryName: "001.jpg", name: "001.jpg" }], + readCbrPage: async () => ({ entryName: "001.jpg", data: Buffer.from([0xff, 0xd8, 0xff, 0xd9]) }) +})); + describe("pdf metadata extraction", () => { it("falls back to file name and reads simple PDF info fields", async () => { const dir = mkdtempSync(join(tmpdir(), "readabook-")); @@ -36,3 +41,16 @@ describe("cbz metadata extraction", () => { expect(pages.map((page) => page.name)).toEqual(["001.jpg", "002.jpg"]); }); }); + +describe("cbr metadata extraction", () => { + it("uses the file name as title and first extracted image as cover", async () => { + const dir = mkdtempSync(join(tmpdir(), "readabook-")); + const file = join(dir, "Comic Two.cbr"); + writeFileSync(file, "rar"); + + const metadata = await extractMetadata(file, dir); + + expect(metadata.title).toBe("Comic Two"); + expect(metadata.coverPath).toMatch(/covers\/[a-f0-9]+\.jpg$/); + }); +}); diff --git a/apps/api/src/scanner/metadata.ts b/apps/api/src/scanner/metadata.ts index ea232f1..b5bd46a 100644 --- a/apps/api/src/scanner/metadata.ts +++ b/apps/api/src/scanner/metadata.ts @@ -3,8 +3,9 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, extname, join } from "node:path"; import AdmZip from "adm-zip"; import { XMLParser } from "fast-xml-parser"; -import { listCbrImageEntries } from "../common/cbr.js"; +import { listCbrImageEntries, readCbrPage } from "../common/cbr.js"; import { listCbzImageEntries } from "../common/cbz.js"; +import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js"; export type BookMetadata = { title: string; @@ -64,7 +65,7 @@ function extractEpubMetadata(filePath: string, storageDir: string): BookMetadata isbn, language: firstText(metadata["dc:language"]), publisher: firstText(metadata["dc:publisher"]), - publishedDate: firstText(metadata["dc:date"]), + publishedDate: normalizePublishedDate(firstText(metadata["dc:date"])), coverPath }; } @@ -97,10 +98,12 @@ function extractCbzMetadata(filePath: string, storageDir: string): BookMetadata } async function extractCbrMetadata(filePath: string, storageDir: string): Promise { - await listCbrImageEntries(filePath); + const firstPage = (await listCbrImageEntries(filePath))[0]; + const page = await readCbrPage(filePath, 1, storageDir); + const coverPath = writeCoverData(page.data, firstPage.entryName, filePath, storageDir); return { ...fallbackMetadata(filePath), - coverPath: null + coverPath }; } @@ -165,6 +168,15 @@ function extractCover(zip: AdmZip, coverPathInZip: string, filePath: string, sto return target; } +function writeCoverData(data: Buffer, entryName: string, filePath: string, storageDir: string): string { + const extension = extname(entryName) || ".jpg"; + const hash = createHash("sha256").update(filePath).digest("hex").slice(0, 24); + const target = join(storageDir, "covers", `${hash}${extension}`); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, data); + return target; +} + function matchPdfInfo(text: string, key: string): string | null { return text.match(new RegExp(`/${key}\\s*\\(([^)]{1,500})\\)`))?.[1] ?? null; } diff --git a/apps/api/src/scanner/scanner.service.test.ts b/apps/api/src/scanner/scanner.service.test.ts index 2238eee..b23b2b8 100644 --- a/apps/api/src/scanner/scanner.service.test.ts +++ b/apps/api/src/scanner/scanner.service.test.ts @@ -1,5 +1,24 @@ -import { describe, expect, it } from "vitest"; -import { scanDigest } from "./scanner.service.js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { DatabaseService } from "../database/database.service.js"; +import { books, libraries } from "../database/schema.js"; +import { JobsService } from "../jobs/jobs.service.js"; +import { enrichmentDigest, preserveExistingBookValues, scanDigest } from "./scanner.service.js"; +import { ScannerService } from "./scanner.service.js"; + +const previousDatabasePath = process.env.DATABASE_PATH; +const previousStorageDir = process.env.STORAGE_DIR; +const tempDirs: string[] = []; + +afterEach(() => { + process.env.DATABASE_PATH = previousDatabasePath; + process.env.STORAGE_DIR = previousStorageDir; + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); describe("scan digest", () => { it("reports incomplete files without exposing huge traces", () => { @@ -17,4 +36,217 @@ describe("scan digest", () => { expect(detail).toContain("broken.cbr"); expect(detail.length).toBeLessThan(380); }); + + it("reports metadata enrichment jobs as enrichment, not scans", () => { + expect(enrichmentDigest(35, [])).toBe("Enriched 35 book(s)"); + }); + + it("does not erase existing metadata or cover when a rescan has less information", () => { + const existing: typeof books.$inferSelect = { + id: 1, + libraryId: 1, + seriesId: null, + title: "Daredevil", + author: "Roy Thomas", + description: "Existing description", + isbn: "9782809476255", + isbn13: "9782809476255", + identifiersJson: null, + localMetadataJson: null, + language: "fre", + publisher: "Panini comics", + publishedDate: "2019", + volumeNumber: null, + volumeLabel: null, + format: "cbz", + filePath: "/library/Daredevil.cbz", + coverPath: "/storage/covers/daredevil.jpg", + metadataStatus: "enriched", + metadataProvenanceJson: JSON.stringify({ author: "bnf", coverPath: "openlibrary" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 12, + fileMtime: "2026-08-23T00:00:00.000Z", + createdAt: "2026-08-23T00:00:00.000Z", + updatedAt: "2026-08-23T00:00:00.000Z" + }; + + const next = preserveExistingBookValues( + { + title: "Daredevil", + author: null, + description: null, + isbn: null, + isbn13: null, + language: null, + publisher: null, + publishedDate: null, + coverPath: null, + metadataStatus: "none", + metadataProvenanceJson: JSON.stringify({ title: "local" }), + scanStatus: "succeeded" as const + }, + existing + ); + + expect(next).toMatchObject({ + author: "Roy Thomas", + description: "Existing description", + isbn: "9782809476255", + coverPath: "/storage/covers/daredevil.jpg", + metadataStatus: "enriched" + }); + expect(JSON.parse(String(next.metadataProvenanceJson))).toMatchObject({ + title: "local", + author: "bnf", + coverPath: "openlibrary" + }); + }); + + it("does not replace an existing valid publication date with a sentinel date", () => { + const existing = { + id: 1, + libraryId: 1, + seriesId: null, + title: "Lord of the Mysteries", + author: null, + description: null, + isbn: null, + isbn13: null, + identifiersJson: null, + localMetadataJson: null, + language: null, + publisher: null, + publishedDate: "2018", + volumeNumber: null, + volumeLabel: null, + format: "epub", + filePath: "/library/Lord of the Mysteries.epub", + coverPath: null, + metadataStatus: "partial", + metadataProvenanceJson: JSON.stringify({ publishedDate: "existing" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 12, + fileMtime: "2026-08-23T00:00:00.000Z", + createdAt: "2026-08-23T00:00:00.000Z", + updatedAt: "2026-08-23T00:00:00.000Z" + } satisfies typeof books.$inferSelect; + + const next = preserveExistingBookValues( + { + title: "Lord of the Mysteries", + publishedDate: "0101-01-01T00:00:00+00:00", + metadataStatus: "partial", + metadataProvenanceJson: JSON.stringify({ title: "local", publishedDate: "openlibrary" }) + }, + existing + ); + + expect(next.publishedDate).toBe("2018"); + }); + + it.runIf(canLoadBetterSqlite())("updates the existing book when an insert races with books.file_path uniqueness", () => { + const database = createDatabase(); + const now = database.now(); + const library = database.db + .insert(libraries) + .values({ name: "Corpus", path: "/library", enabled: true, createdAt: now, updatedAt: now }) + .returning() + .get(); + database.db + .insert(books) + .values({ + libraryId: library.id, + seriesId: null, + title: "Daredevil", + author: null, + description: null, + isbn: null, + isbn13: null, + identifiersJson: null, + localMetadataJson: null, + language: null, + publisher: null, + publishedDate: null, + volumeNumber: null, + volumeLabel: null, + format: "cbz", + filePath: "/library/Daredevil.cbz", + coverPath: null, + metadataStatus: "none", + metadataProvenanceJson: null, + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 1, + fileMtime: now, + createdAt: now, + updatedAt: now + }) + .run(); + const scanner = new ScannerService(database, new JobsService(database), {} as never); + const values: Omit = { + libraryId: library.id, + seriesId: null, + title: "Daredevil", + author: "Roy Thomas", + description: "Updated metadata", + isbn: null, + isbn13: null, + identifiersJson: null, + localMetadataJson: null, + language: null, + publisher: null, + publishedDate: null, + volumeNumber: null, + volumeLabel: null, + format: "cbz", + filePath: "/library/Daredevil.cbz", + coverPath: "/storage/covers/daredevil.jpg", + metadataStatus: "enriched", + metadataProvenanceJson: JSON.stringify({ author: "bnf", coverPath: "local" }), + scanStatus: "succeeded", + enrichmentStatus: "succeeded", + fileSize: 2, + fileMtime: now, + updatedAt: now + }; + + (scanner as unknown as { + upsertBookByFilePath(values: Omit, existing: undefined, createdAt: string): void; + }).upsertBookByFilePath( + values, + undefined, + now + ); + + const rows = database.db.select().from(books).all(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + author: "Roy Thomas", + coverPath: "/storage/covers/daredevil.jpg", + metadataStatus: "enriched", + fileSize: 2 + }); + + database.onModuleDestroy(); + }); }); + +function createDatabase(): DatabaseService { + const dir = mkdtempSync(join(tmpdir(), "readabook-scanner-service-")); + tempDirs.push(dir); + process.env.DATABASE_PATH = join(dir, "readabook.sqlite"); + process.env.STORAGE_DIR = join(dir, "storage"); + return new DatabaseService(); +} + +function canLoadBetterSqlite(): boolean { + try { + const database = createDatabase(); + database.onModuleDestroy(); + return true; + } catch { + return false; + } +} diff --git a/apps/api/src/scanner/scanner.service.ts b/apps/api/src/scanner/scanner.service.ts index b526caa..33382b6 100644 --- a/apps/api/src/scanner/scanner.service.ts +++ b/apps/api/src/scanner/scanner.service.ts @@ -3,9 +3,11 @@ import { existsSync, readdirSync, statSync } from "node:fs"; import { basename, extname, join } from "node:path"; import { eq, inArray } from "drizzle-orm"; import { DatabaseService } from "../database/database.service.js"; -import { automationSettings, books, libraries } from "../database/schema.js"; +import { automationSettings, books, libraries, series } from "../database/schema.js"; import { JobsService } from "../jobs/jobs.service.js"; import { MetadataService } from "../metadata/metadata.service.js"; +import { extractSeriesVolume } from "../metadata/use-cases/extract-series-volume.js"; +import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js"; import { extractMetadata } from "./metadata.js"; @Injectable() @@ -86,23 +88,40 @@ export class ScannerService { this.jobs.markRunning(jobId, "Enriching existing books"); const rows = this.database.db.select({ id: books.id }).from(books).all(); let count = 0; + const failures: ScanFailure[] = []; for (const row of rows) { - await this.metadata.enrichBook(row.id); - count += 1; + this.markBookEnrichmentStatus(row.id, "running"); + try { + await this.metadata.enrichBook(row.id); + this.markBookEnrichmentStatus(row.id, "succeeded"); + count += 1; + } catch (error) { + this.markBookEnrichmentStatus(row.id, "failed"); + failures.push({ filePath: `book #${row.id}`, error: errorMessage(error) }); + } } - this.jobs.markSucceeded(jobId, `Enriched ${count} book(s)`); + this.jobs.markSucceeded(jobId, enrichmentDigest(count, failures)); } private async ingestFile(libraryId: number, filePath: string): Promise { const stats = statSync(filePath); + const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get(); + if (existing) { + this.database.db + .update(books) + .set({ scanStatus: "running", enrichmentStatus: "running", updatedAt: this.database.now() }) + .where(eq(books.id, existing.id)) + .run(); + } const localMetadata = await extractMetadata(filePath, this.database.config.storageDir); const now = this.database.now(); const format = bookFormatFromPath(filePath); - const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get(); const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true; const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich }); + const seriesInfo = this.resolveSeries(metadata.title, filePath); const values = { libraryId, + seriesId: seriesInfo.seriesId, title: metadata.title, author: metadata.author, description: metadata.description, @@ -113,25 +132,31 @@ export class ScannerService { language: metadata.language, publisher: metadata.publisher, publishedDate: metadata.publishedDate, + volumeNumber: seriesInfo.volumeNumber, + volumeLabel: seriesInfo.volumeLabel, format, filePath, coverPath: metadata.coverPath, + metadataStatus: metadata.metadataStatus, + metadataProvenanceJson: metadata.metadataProvenanceJson, + scanStatus: "succeeded" as const, + enrichmentStatus: shouldRemoteEnrich ? ("succeeded" as const) : ("idle" as const), fileSize: stats.size, fileMtime: stats.mtime.toISOString(), updatedAt: now }; - existing - ? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get() - : this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get(); + this.upsertBookByFilePath(values, existing, now); } private ingestIncompleteFile(libraryId: number, filePath: string): void { const stats = statSync(filePath); const now = this.database.now(); - const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get(); + const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get(); + const seriesInfo = this.resolveSeries(basename(filePath, extname(filePath)), filePath); const values = { libraryId, + seriesId: seriesInfo.seriesId, title: basename(filePath, extname(filePath)), author: null, description: null, @@ -154,17 +179,41 @@ export class ScannerService { language: null, publisher: null, publishedDate: null, + volumeNumber: seriesInfo.volumeNumber, + volumeLabel: seriesInfo.volumeLabel, format: bookFormatFromPath(filePath), filePath, coverPath: null, + metadataStatus: "none" as const, + metadataProvenanceJson: JSON.stringify({ title: "local" }), + scanStatus: "failed" as const, + enrichmentStatus: "failed" as const, fileSize: stats.size, fileMtime: stats.mtime.toISOString(), updatedAt: now }; - existing - ? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get() - : this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get(); + this.upsertBookByFilePath(values, existing, now); + } + + private upsertBookByFilePath( + values: Omit, + existing: typeof books.$inferSelect | undefined, + createdAt: string + ): void { + if (existing) { + this.database.db.update(books).set(preserveExistingBookValues(values, existing)).where(eq(books.id, existing.id)).run(); + return; + } + try { + this.database.db.insert(books).values({ ...values, createdAt }).run(); + return; + } catch (error) { + if (!isUniqueFilePathError(error)) throw error; + const current = this.database.db.select().from(books).where(eq(books.filePath, values.filePath)).get(); + if (!current) throw error; + this.database.db.update(books).set(preserveExistingBookValues(values, current)).where(eq(books.id, current.id)).run(); + } } private removeMissingBooks(libraryId: number, seen: Set): number { @@ -184,6 +233,32 @@ export class ScannerService { .get()?.autoEnrichNewBooks ); } + + private markBookEnrichmentStatus(id: number, enrichmentStatus: "running" | "succeeded" | "failed"): void { + this.database.db.update(books).set({ enrichmentStatus, updatedAt: this.database.now() }).where(eq(books.id, id)).run(); + } + + private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } { + const parsed = extractSeriesVolume(title, filePath); + const now = this.database.now(); + const row = this.database.db + .insert(series) + .values({ + title: parsed.seriesTitle, + normalizedTitle: parsed.normalizedSeriesTitle, + description: null, + publisher: null, + createdAt: now, + updatedAt: now + }) + .onConflictDoUpdate({ + target: series.normalizedTitle, + set: { title: parsed.seriesTitle, updatedAt: now } + }) + .returning({ id: series.id }) + .get(); + return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel }; + } } type ScanFailure = { @@ -202,6 +277,17 @@ export function scanDigest(scanned: number, removed: number, failures: ScanFailu return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`; } +export function enrichmentDigest(enriched: number, failures: ScanFailure[]): string { + const base = `Enriched ${enriched} book(s)`; + if (!failures.length) return base; + const examples = failures + .slice(0, 3) + .map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`) + .join("; "); + const extra = failures.length > 3 ? `; ${failures.length - 3} more` : ""; + return `${base}, ${failures.length} incomplete book(s): ${examples}${extra}`; +} + function errorMessage(error: unknown): string { if (error instanceof Error && error.message) return truncate(error.message); return truncate(String(error)); @@ -233,3 +319,49 @@ function bookFormatFromPath(filePath: string): "epub" | "pdf" | "cbz" | "cbr" { if (extension === ".cbr") return "cbr"; return "pdf"; } + +export function preserveExistingBookValues>(values: T, existing: typeof books.$inferSelect): T { + const next = { ...values }; + if (next.scanStatus === "failed" && existing.title) { + next.title = existing.title as never; + } + for (const field of ["author", "description", "isbn", "isbn13", "language", "publisher", "publishedDate", "coverPath", "seriesId", "volumeNumber", "volumeLabel"] as const) { + if (field === "publishedDate") { + next.publishedDate = (normalizePublishedDate(next.publishedDate) ?? normalizePublishedDate(existing.publishedDate)) as never; + continue; + } + if (next[field] == null && existing[field] != null) { + next[field] = existing[field] as never; + } + } + next.metadataStatus = computeMetadataStatus(next, existing.metadataStatus) as never; + next.metadataProvenanceJson = mergeProvenanceJson(String(next.metadataProvenanceJson ?? "{}"), existing.metadataProvenanceJson) as never; + return next; +} + +function computeMetadataStatus(values: Partial, existingStatus: string): "enriched" | "partial" | "none" { + const hasCover = Boolean(values.coverPath); + const filled = [values.author, values.description, values.isbn, values.language, values.publisher, values.publishedDate].filter(Boolean).length; + const computed = hasCover && filled >= 2 ? "enriched" : hasCover || filled > 0 ? "partial" : "none"; + const rank = { none: 0, partial: 1, enriched: 2 } as const; + const safeExisting = existingStatus === "enriched" || existingStatus === "partial" || existingStatus === "none" ? existingStatus : "none"; + return rank[computed] >= rank[safeExisting] ? computed : safeExisting; +} + +function mergeProvenanceJson(nextJson: string, existingJson: string | null): string { + return JSON.stringify({ ...parseJsonObject(existingJson), ...parseJsonObject(nextJson) }); +} + +function parseJsonObject(value: string | null): Record { + if (!value) return {}; + try { + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record) : {}; + } catch { + return {}; + } +} + +function isUniqueFilePathError(error: unknown): boolean { + return error instanceof Error && error.message.includes("UNIQUE constraint failed: books.file_path"); +} diff --git a/apps/web/src/pages/AdminAutomationPage.tsx b/apps/web/src/pages/AdminAutomationPage.tsx index a570abf..840feef 100644 --- a/apps/web/src/pages/AdminAutomationPage.tsx +++ b/apps/web/src/pages/AdminAutomationPage.tsx @@ -4,16 +4,21 @@ import type { AutomationFrequency, AutomationScheduleDto, AutomationSettingsDto, - MetadataProviderId, MetadataSourcesConfigDto } from "@readabook/shared"; import { api, getApiFallback } from "../api/client"; import { ErrorRibbon, LoadingState, Panel } from "../components/ui"; import { + type AdminMetadataProviderId, + type AdminMetadataSourcesConfig, + defaultMetadataSources, metadataSourcesPayload, moveSource, normalizeMetadataSources, providerLabels, + providerUiMessage, + providerUiState, + providerUiStateLabel, scheduleDays, scheduleSummary } from "./adminAutomation"; @@ -28,14 +33,9 @@ type ApiState = { success?: string; }; -const defaultMetadataConfig: MetadataSourcesConfigDto = { +const defaultMetadataConfig: AdminMetadataSourcesConfig = { isbnPriorityEnabled: true, - sources: [ - { provider: "local", enabled: true, priority: 0, hasApiKey: false }, - { provider: "openlibrary", enabled: false, priority: 1, hasApiKey: false }, - { provider: "googlebooks", enabled: false, priority: 2, hasApiKey: false }, - { provider: "bnf", enabled: false, priority: 3, hasApiKey: false } - ] + sources: defaultMetadataSources }; const defaultAutomationSettings: AutomationSettingsDto = { @@ -47,7 +47,7 @@ const defaultAutomationSettings: AutomationSettingsDto = { export function AdminAutomationPage() { const [tab, setTab] = useState("sources"); - const [metadataState, setMetadataState] = useState>({ + const [metadataState, setMetadataState] = useState>({ initial: null, draft: null, loading: true, @@ -59,7 +59,7 @@ export function AdminAutomationPage() { loading: true, saving: false }); - const [apiKeys, setApiKeys] = useState>>({}); + const [apiKeys, setApiKeys] = useState>>({}); const metadataDirty = useMemo( () => Boolean(metadataState.initial && metadataState.draft && JSON.stringify(metadataState.initial) !== JSON.stringify(metadataState.draft)), @@ -78,7 +78,7 @@ export function AdminAutomationPage() { setMetadataState({ initial: next, draft: next, loading: false, saving: false }); setApiKeys({}); } catch (error) { - const fallback = getApiFallback(error); + const fallback = getApiFallback(error); const next = normalizeMetadataSources(fallback ?? defaultMetadataConfig); setMetadataState({ initial: next, @@ -226,11 +226,11 @@ function MetadataSourcesPanel({ onSubmit, onRefresh }: { - state: ApiState; + state: ApiState; dirty: boolean; - apiKeys: Partial>; - setApiKeys: (next: Partial>) => void; - onChange: (draft: MetadataSourcesConfigDto) => void; + apiKeys: Partial>; + setApiKeys: (next: Partial>) => void; + onChange: (draft: AdminMetadataSourcesConfig) => void; onSubmit: (event: FormEvent) => void; onRefresh: () => Promise; }) { @@ -297,14 +297,25 @@ function MetadataSourcesPanel({ {source.enabled ? "active" : "inactive"} - +
+
+ {providerUiStateLabel(source)} + {providerUiMessage(source)} +
+ {source.provider === "comicvine" && ( +

+ Usage non commercial uniquement. Verifier la compatibilite avec l'usage de ReadaBook. +

+ )} + +