feat(api,web): providers ComicVine + MangaDex et pilotage de l'enrichissement
- Adaptateurs comic-vine et mangadex avec helper de fetch partagé, timeouts et fallback durcis sur les providers existants - Scoring des correspondances amélioré, normalisation de la date de publication, chaîne de résolution des providers étendue - Scanner : statuts par livre (scan/enrichissement) et page admin d'automatisation alignée Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Git Agent
@ -2,6 +2,8 @@ import { Injectable } from "@nestjs/common";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch, providerHttpError } from "./provider-fetch.js";
|
||||
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true });
|
||||
|
||||
@ -36,8 +38,8 @@ export class BnfProvider implements MetadataProvider {
|
||||
url.searchParams.set("query", query);
|
||||
url.searchParams.set("maximumRecords", String(maximumRecords));
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
||||
if (!response.ok) return [];
|
||||
const response = await providerFetch(this.id, url, { timeoutMs: 5000 });
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `BnF HTTP ${response.status}`);
|
||||
const parsed = parser.parse(await response.text());
|
||||
const records = asArray(parsed?.searchRetrieveResponse?.records?.record)
|
||||
.map((entry) => (entry.recordData as Record<string, unknown> | undefined)?.record)
|
||||
@ -53,7 +55,7 @@ export class BnfProvider implements MetadataProvider {
|
||||
isbn: bestIsbn(fields, expectedIsbn13),
|
||||
language: subfield(fields, "101", "a"),
|
||||
publisher: subfield(fields, "210", "c") ?? subfield(fields, "214", "c"),
|
||||
publishedDate: cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d"))
|
||||
publishedDate: normalizePublishedDate(cleanDate(subfield(fields, "210", "d") ?? subfield(fields, "214", "d")))
|
||||
};
|
||||
})
|
||||
.sort((left, right) => Number(Boolean(right.isbn)) - Number(Boolean(left.isbn)));
|
||||
|
||||
160
apps/api/src/metadata/adapters/comic-vine.provider.ts
Normal file
160
apps/api/src/metadata/adapters/comic-vine.provider.ts
Normal file
@ -0,0 +1,160 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class ComicVineProvider implements MetadataProvider {
|
||||
readonly id = "comicvine" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
assertApiKey(config);
|
||||
const title = cleanComicTitle(query.title);
|
||||
const relaxed = title.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim();
|
||||
const matches = [
|
||||
...(await searchComicVine("volume", title, config)),
|
||||
...(await searchComicVine("issue", title, config)),
|
||||
...(relaxed && relaxed !== title ? await searchComicVine("volume", relaxed, config) : [])
|
||||
];
|
||||
return rankMatches(query.title, dedupe(matches));
|
||||
}
|
||||
}
|
||||
|
||||
export class ComicVineProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "missing-key" | "invalid-key" | "rate-limit" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ComicVineProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function searchComicVine(resource: "volume" | "issue", title: string, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://comicvine.gamespot.com/api/search/");
|
||||
url.searchParams.set("api_key", config.apiKey!);
|
||||
url.searchParams.set("format", "json");
|
||||
url.searchParams.set("resources", resource);
|
||||
url.searchParams.set("query", title);
|
||||
url.searchParams.set("limit", "10");
|
||||
url.searchParams.set(
|
||||
"field_list",
|
||||
resource === "volume" ? "id,name,description,image,start_year,publisher" : "id,name,description,image,cover_date,store_date,volume"
|
||||
);
|
||||
const response = await providerFetch("comicvine", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (Comic Vine; non-commercial)" },
|
||||
timeoutMs: 6000
|
||||
});
|
||||
const data = (await parseComicVineResponse(response)) as { results?: Array<Record<string, unknown>> };
|
||||
return (data.results ?? []).map((entry) => comicVineToMatch(resource, entry));
|
||||
}
|
||||
|
||||
function assertApiKey(config: MetadataProviderConfig): void {
|
||||
if (!config.apiKey?.trim()) {
|
||||
throw new ComicVineProviderError("missing-key", 0, "Comic Vine API key is required");
|
||||
}
|
||||
}
|
||||
|
||||
async function parseComicVineResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { status_code?: number; error?: string };
|
||||
if (response.status === 429) throw new ComicVineProviderError("rate-limit", response.status, data.error ?? "Comic Vine rate limit");
|
||||
if (response.status === 401 || response.status === 403) throw new ComicVineProviderError("invalid-key", response.status, data.error ?? "Comic Vine API key rejected");
|
||||
if (!response.ok) throw new ComicVineProviderError("http", response.status, data.error ?? `Comic Vine HTTP ${response.status}`);
|
||||
if (data.status_code && data.status_code !== 1) {
|
||||
const code = data.status_code === 100 || data.status_code === 101 ? "invalid-key" : "http";
|
||||
throw new ComicVineProviderError(code, 200, data.error ?? `Comic Vine status ${data.status_code}`);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function comicVineToMatch(resource: "volume" | "issue", entry: Record<string, unknown>): MetadataMatch & { comicVineRank?: number } {
|
||||
const volume = objectValue(entry.volume);
|
||||
const title = resource === "issue" ? [stringValue(volume.name), stringValue(entry.name)].filter(Boolean).join(" ") : stringValue(entry.name);
|
||||
return {
|
||||
title: title || undefined,
|
||||
description: cleanHtml(stringValue(entry.description)),
|
||||
publisher: stringValue(objectValue(entry.publisher).name),
|
||||
publishedDate: normalizePublishedDate(resource === "volume" ? stringValue(entry.start_year) : yearFromDate(stringValue(entry.cover_date) ?? stringValue(entry.store_date))),
|
||||
coverUrl: imageUrl(entry.image),
|
||||
sourceId: stringValue(entry.id)
|
||||
};
|
||||
}
|
||||
|
||||
function cleanComicTitle(value: string): string {
|
||||
return value
|
||||
.replace(/\.[A-Za-z0-9]{2,5}$/g, " ")
|
||||
.replace(/[._]+/g, " ")
|
||||
.replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ")
|
||||
.replace(/\([^)]*\)/g, " ")
|
||||
.replace(/\b(e?bdz|Paprika\+?|emuleCenter(?:\.|\s+)net)\b/gi, " ")
|
||||
.replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/[+]+/g, " ")
|
||||
.replace(/\s+-\s+/g, " ")
|
||||
.replace(/\s*-\s*$/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function rankMatches(originalTitle: string, matches: Array<MetadataMatch & { comicVineRank?: number }>): MetadataMatch[] {
|
||||
return [...matches]
|
||||
.map((match) => ({ ...match, comicVineRank: comicRank(originalTitle, match) }))
|
||||
.sort((left, right) => (right.comicVineRank ?? 0) - (left.comicVineRank ?? 0))
|
||||
.map(({ comicVineRank: _rank, ...match }) => match);
|
||||
}
|
||||
|
||||
function comicRank(originalTitle: string, match: MetadataMatch): number {
|
||||
let rank = tokenOverlap(cleanComicTitle(originalTitle), match.title ?? "") * 10;
|
||||
if (match.coverUrl) rank += 1;
|
||||
if (match.description) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function dedupe(matches: MetadataMatch[]): MetadataMatch[] {
|
||||
const seen = new Set<string>();
|
||||
return matches.filter((match) => {
|
||||
const key = [match.sourceId, match.title].filter(Boolean).join("|");
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function cleanHtml(value: string | null): string | null {
|
||||
if (!value) return null;
|
||||
return value.replace(/<[^>]*>/g, " ").replace(/ /g, " ").replace(/&/g, "&").replace(/\s+/g, " ").trim() || null;
|
||||
}
|
||||
|
||||
function imageUrl(value: unknown): string | null {
|
||||
const image = objectValue(value);
|
||||
return stringValue(image.original_url) ?? stringValue(image.super_url) ?? stringValue(image.medium_url) ?? stringValue(image.small_url);
|
||||
}
|
||||
|
||||
function yearFromDate(value: string | null): string | null {
|
||||
return value?.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTokens(left));
|
||||
const rightTokens = new Set(normalizeTokens(right));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTokens(value: string): string[] {
|
||||
return value.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(" ").filter(Boolean);
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
if (typeof value === "number") return String(value);
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class GoogleBooksProvider implements MetadataProvider {
|
||||
@ -12,54 +14,157 @@ export class GoogleBooksProvider implements MetadataProvider {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
const query = `isbn:${isbn}`;
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("maxResults", "1");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
if (!response.ok) return null;
|
||||
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
|
||||
const info = data.items?.[0]?.volumeInfo;
|
||||
if (!info) return null;
|
||||
return {
|
||||
title: stringValue(info.title) ?? undefined,
|
||||
author: arrayJoin(info.authors),
|
||||
description: stringValue(info.description),
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, lookup.identifiers.isbn13)
|
||||
};
|
||||
const matches = await this.searchVolumes(`isbn:${isbn}`, config, lookup.title, lookup.author, lookup.identifiers.isbn13);
|
||||
return rankMatches(lookup.title, matches)[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const attempts = googleBookQueries(query);
|
||||
const matches: MetadataMatch[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const attempt of attempts) {
|
||||
for (const match of await this.searchVolumes(attempt, config, query.title, query.author, query.isbn ? toIsbn13(query.isbn) : null)) {
|
||||
const key = [match.sourceId, match.isbn, match.title, match.author].filter(Boolean).join("|");
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
matches.push(match);
|
||||
}
|
||||
}
|
||||
return rankMatches(query.title, matches);
|
||||
}
|
||||
|
||||
private async searchVolumes(
|
||||
googleQuery: string,
|
||||
config: MetadataProviderConfig,
|
||||
originalTitle: string,
|
||||
originalAuthor: string | null,
|
||||
expectedIsbn13: string | null
|
||||
): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://www.googleapis.com/books/v1/volumes");
|
||||
url.searchParams.set("q", `intitle:${query.title}${query.author ? `+inauthor:${query.author}` : ""}`);
|
||||
url.searchParams.set("maxResults", "5");
|
||||
url.searchParams.set("q", googleQuery);
|
||||
url.searchParams.set("maxResults", "10");
|
||||
url.searchParams.set("printType", "books");
|
||||
if (config.apiKey) url.searchParams.set("key", config.apiKey);
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
|
||||
if (!response.ok) return [];
|
||||
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
|
||||
const response = await providerFetch(this.id, url, { timeoutMs: 4000 });
|
||||
const data = (await parseGoogleResponse(response)) as { items?: Array<{ id?: string; volumeInfo?: Record<string, unknown> }> };
|
||||
return (data.items ?? [])
|
||||
.map((item) => item.volumeInfo)
|
||||
.filter((info): info is Record<string, unknown> => Boolean(info))
|
||||
.map((info) => ({
|
||||
.map((item) => ({ sourceId: item.id, info: item.volumeInfo }))
|
||||
.filter((item): item is { sourceId: string | undefined; info: Record<string, unknown> } => Boolean(item.info))
|
||||
.map(({ sourceId, info }) => ({
|
||||
title: stringValue(info.title) ?? undefined,
|
||||
author: arrayJoin(info.authors),
|
||||
description: stringValue(info.description),
|
||||
language: stringValue(info.language),
|
||||
publisher: stringValue(info.publisher),
|
||||
publishedDate: stringValue(info.publishedDate),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
publishedDate: normalizePublishedDate(stringValue(info.publishedDate)),
|
||||
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers, expectedIsbn13),
|
||||
coverUrl: coverUrl(info.imageLinks),
|
||||
sourceId,
|
||||
identifiers: { candidates: isbnCandidates(info.industryIdentifiers) },
|
||||
googleRank: googleRank(originalTitle, originalAuthor, info)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
export class GoogleBooksProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "quota" | "auth" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "GoogleBooksProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function parseGoogleResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { error?: { message?: string; status?: string } };
|
||||
if (response.ok) return data;
|
||||
const message = data.error?.message ?? `Google Books HTTP ${response.status}`;
|
||||
if (response.status === 429) throw new GoogleBooksProviderError("quota", response.status, message);
|
||||
if (response.status === 401 || response.status === 403) throw new GoogleBooksProviderError("auth", response.status, message);
|
||||
throw new GoogleBooksProviderError("http", response.status, message);
|
||||
}
|
||||
|
||||
function googleBookQueries(query: MetadataSearchQuery): string[] {
|
||||
const cleaned = cleanGoogleBooksTitle(query.title);
|
||||
const relaxed = relaxSeriesTitle(cleaned);
|
||||
return [
|
||||
query.isbn ? `isbn:${query.isbn}` : null,
|
||||
googleTitleQuery(cleaned, query.author, true),
|
||||
googleTitleQuery(cleaned, query.author, false),
|
||||
relaxed !== cleaned ? googleTitleQuery(relaxed, query.author, true) : null,
|
||||
relaxed !== cleaned ? googleTitleQuery(relaxed, null, false) : null,
|
||||
googleTitleQuery(cleaned, null, false)
|
||||
].filter((value, index, values): value is string => Boolean(value) && values.indexOf(value) === index);
|
||||
}
|
||||
|
||||
function googleTitleQuery(title: string, author: string | null, quoted: boolean): string {
|
||||
const titlePart = quoted ? `intitle:"${title.replace(/"/g, " ")}"` : `intitle:${title}`;
|
||||
return author ? `${titlePart}+inauthor:${author}` : titlePart;
|
||||
}
|
||||
|
||||
function cleanGoogleBooksTitle(value: string): string {
|
||||
return value
|
||||
.replace(/\.[A-Za-z0-9]{2,5}$/g, " ")
|
||||
.replace(/[._]+/g, " ")
|
||||
.replace(/\b(FRENCH|TRUEFRENCH|MULTI|CBZ|CBR|EPUB|PDF|eBook|ebook|scan|digital)\b/gi, " ")
|
||||
.replace(/\b(e?bdz|Paprika\+?|emuleCenter\.net)\b/gi, " ")
|
||||
.replace(/\bT(?:ome)?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/\bVol(?:ume)?\.?\s*0?(\d{1,3})\b/gi, " $1 ")
|
||||
.replace(/[+]+/g, " ")
|
||||
.replace(/\s+-\s+/g, " ")
|
||||
.replace(/\s*-\s*$/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function relaxSeriesTitle(value: string): string {
|
||||
return value.replace(/\b\d{1,3}\b/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function rankMatches(originalTitle: string, matches: Array<MetadataMatch & { googleRank?: number }>): MetadataMatch[] {
|
||||
return [...matches]
|
||||
.sort((left, right) => (right.googleRank ?? 0) - (left.googleRank ?? 0))
|
||||
.map(({ googleRank: _googleRank, ...match }) => match);
|
||||
}
|
||||
|
||||
function googleRank(originalTitle: string, originalAuthor: string | null, info: Record<string, unknown>): number {
|
||||
const expectedVolume = volumeNumber(originalTitle);
|
||||
const candidateTitle = [stringValue(info.title), stringValue(info.subtitle)].filter(Boolean).join(" ");
|
||||
let rank = tokenOverlap(cleanGoogleBooksTitle(originalTitle), candidateTitle) * 10;
|
||||
if (expectedVolume) {
|
||||
const candidateVolume = volumeNumber(candidateTitle);
|
||||
rank += candidateVolume === expectedVolume ? 6 : candidateVolume ? -4 : 0;
|
||||
}
|
||||
if (originalAuthor && arrayJoin(info.authors)?.toLowerCase().includes(originalAuthor.toLowerCase())) rank += 2;
|
||||
if (stringValue(info.description)) rank += 1;
|
||||
if (coverUrl(info.imageLinks)) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function volumeNumber(value: string): string | null {
|
||||
return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTokens(left));
|
||||
const rightTokens = new Set(normalizeTokens(right));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTokens(value: string): string[] {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.split(" ")
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -77,3 +182,20 @@ function isbnFromIndustryIdentifiers(value: unknown, expectedIsbn13: string | nu
|
||||
const isbn10 = entries.find((entry) => entry.type === "ISBN_10")?.identifier;
|
||||
return stringValue(isbn13) ?? stringValue(isbn10);
|
||||
}
|
||||
|
||||
function isbnCandidates(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map((entry) => stringValue((entry as { identifier?: unknown }).identifier)).filter((entry): entry is string => Boolean(entry));
|
||||
}
|
||||
|
||||
function coverUrl(value: unknown): string | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const links = value as Record<string, unknown>;
|
||||
return (
|
||||
stringValue(links.extraLarge) ??
|
||||
stringValue(links.large) ??
|
||||
stringValue(links.medium) ??
|
||||
stringValue(links.thumbnail) ??
|
||||
stringValue(links.smallThumbnail)
|
||||
)?.replace(/^http:/, "https:") ?? null;
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
|
||||
@Injectable()
|
||||
export class LocalMetadataProvider implements MetadataProvider {
|
||||
@ -20,7 +21,7 @@ export class LocalMetadataProvider implements MetadataProvider {
|
||||
title: query.title,
|
||||
author: query.author,
|
||||
isbn: query.isbn ?? null,
|
||||
publishedDate: query.year ?? null
|
||||
publishedDate: normalizePublishedDate(query.year)
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
179
apps/api/src/metadata/adapters/mangadex.provider.ts
Normal file
179
apps/api/src/metadata/adapters/mangadex.provider.ts
Normal file
@ -0,0 +1,179 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { extractSeriesVolume } from "../use-cases/extract-series-volume.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class MangaDexProvider implements MetadataProvider {
|
||||
readonly id = "mangadex" as const;
|
||||
|
||||
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
|
||||
const matches = await this.searchByMetadata({ title: lookup.title, author: lookup.author, year: lookup.local.year }, config);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
async searchByMetadata(query: MetadataSearchQuery, config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const titles = mangaDexTitleQueries(query.title);
|
||||
const matches: Array<MetadataMatch & { mangaDexRank?: number }> = [];
|
||||
const seen = new Map<string, number>();
|
||||
for (const title of titles) {
|
||||
for (const match of await searchManga(title, config)) {
|
||||
const key = match.sourceId ?? `${match.title}|${match.author}`;
|
||||
const ranked = { ...match, scoreTitle: title, mangaDexRank: mangaRank(query.title, title, match) };
|
||||
const existingIndex = seen.get(key);
|
||||
if (existingIndex === undefined) {
|
||||
seen.set(key, matches.length);
|
||||
matches.push(ranked);
|
||||
continue;
|
||||
}
|
||||
if ((ranked.mangaDexRank ?? 0) > (matches[existingIndex]?.mangaDexRank ?? 0)) {
|
||||
matches[existingIndex] = ranked;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rankMatches(matches);
|
||||
}
|
||||
}
|
||||
|
||||
export class MangaDexProviderError extends Error {
|
||||
constructor(
|
||||
readonly code: "rate-limit" | "http",
|
||||
readonly status: number,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "MangaDexProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
async function searchManga(title: string, _config: MetadataProviderConfig): Promise<MetadataMatch[]> {
|
||||
const url = new URL("https://api.mangadex.org/manga");
|
||||
url.searchParams.set("title", title);
|
||||
url.searchParams.set("limit", "10");
|
||||
url.searchParams.set("includes[]", "cover_art");
|
||||
url.searchParams.append("includes[]", "author");
|
||||
url.searchParams.append("includes[]", "artist");
|
||||
url.searchParams.set("contentRating[]", "safe");
|
||||
url.searchParams.append("contentRating[]", "suggestive");
|
||||
let response = await providerFetch("mangadex", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" },
|
||||
timeoutMs: 5000
|
||||
});
|
||||
if (response.status === 429) {
|
||||
await sleep(retryDelayMs(response));
|
||||
response = await providerFetch("mangadex", url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 self-hosted metadata provider (MangaDex)" },
|
||||
timeoutMs: 5000
|
||||
});
|
||||
}
|
||||
const data = (await parseMangaDexResponse(response)) as { data?: Array<Record<string, unknown>> };
|
||||
return (data.data ?? []).map(mangaToMatch);
|
||||
}
|
||||
|
||||
function retryDelayMs(response: Response): number {
|
||||
const retryAfter = Number(response.headers.get("Retry-After"));
|
||||
return Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter * 1000, 2000) : 250;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function parseMangaDexResponse(response: Response): Promise<unknown> {
|
||||
const data = (await response.json().catch(() => ({}))) as { errors?: Array<{ detail?: string; title?: string }> };
|
||||
if (response.ok) return data;
|
||||
const message = data.errors?.map((error) => error.detail ?? error.title).filter(Boolean).join("; ") || `MangaDex HTTP ${response.status}`;
|
||||
if (response.status === 429) throw new MangaDexProviderError("rate-limit", response.status, message);
|
||||
throw new MangaDexProviderError("http", response.status, message);
|
||||
}
|
||||
|
||||
function mangaToMatch(manga: Record<string, unknown>): MetadataMatch & { mangaDexRank?: number } {
|
||||
const id = stringValue(manga.id);
|
||||
const attributes = objectValue(manga.attributes);
|
||||
const relationships = Array.isArray(manga.relationships) ? (manga.relationships as Array<Record<string, unknown>>) : [];
|
||||
const cover = relationships.find((entry) => entry.type === "cover_art");
|
||||
const coverFile = stringValue(objectValue(cover?.attributes).fileName);
|
||||
return {
|
||||
title: localizedText(attributes.title) ?? undefined,
|
||||
author: relationshipNames(relationships),
|
||||
description: localizedText(attributes.description),
|
||||
publishedDate: normalizePublishedDate(stringValue(attributes.year)),
|
||||
language: stringValue(attributes.originalLanguage),
|
||||
coverUrl: id && coverFile ? `https://uploads.mangadex.org/covers/${id}/${coverFile}.512.jpg` : null,
|
||||
sourceId: id
|
||||
};
|
||||
}
|
||||
|
||||
function mangaDexTitleQueries(title: string): string[] {
|
||||
const cleaned = extractSeriesVolume(title).seriesTitle;
|
||||
return [cleaned, ...mangaDexTitleAliases(cleaned)].filter(
|
||||
(value, index, values): value is string => Boolean(value) && values.indexOf(value) === index
|
||||
);
|
||||
}
|
||||
|
||||
function mangaDexTitleAliases(title: string): string[] {
|
||||
const normalized = normalizeTitle(title);
|
||||
if (normalized === "demon slayer school days") return ["Demon Slayer Kimetsu Academy", "Kimetsu Academy"];
|
||||
return [];
|
||||
}
|
||||
|
||||
function mangaRank(originalTitle: string, searchedTitle: string, match: MetadataMatch): number {
|
||||
const expectedVolume = volumeNumber(originalTitle);
|
||||
let rank = tokenOverlap(searchedTitle, match.title ?? "") * 10;
|
||||
if (expectedVolume) {
|
||||
const candidateVolume = volumeNumber(match.title ?? "");
|
||||
rank += candidateVolume === expectedVolume ? 4 : candidateVolume ? -2 : 0;
|
||||
}
|
||||
if (match.coverUrl) rank += 1;
|
||||
if (match.description) rank += 1;
|
||||
return rank;
|
||||
}
|
||||
|
||||
function rankMatches(matches: Array<MetadataMatch & { mangaDexRank?: number }>): MetadataMatch[] {
|
||||
return [...matches].sort((left, right) => (right.mangaDexRank ?? 0) - (left.mangaDexRank ?? 0)).map(({ mangaDexRank: _rank, ...match }) => match);
|
||||
}
|
||||
|
||||
function volumeNumber(value: string): string | null {
|
||||
return value.match(/\b(?:T|tome|vol(?:ume)?\.?)\s*0?(\d{1,3})\b/i)?.[1] ?? value.match(/\b0?(\d{1,3})\b/)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function tokenOverlap(left: string, right: string): number {
|
||||
const leftTokens = new Set(normalizeTitle(left).split(" ").filter(Boolean));
|
||||
const rightTokens = new Set(normalizeTitle(right).split(" ").filter(Boolean));
|
||||
if (!leftTokens.size || !rightTokens.size) return 0;
|
||||
return [...leftTokens].filter((token) => rightTokens.has(token)).length / leftTokens.size;
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
return value
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function localizedText(value: unknown): string | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const entries = value as Record<string, unknown>;
|
||||
return stringValue(entries.en) ?? stringValue(entries.fr) ?? Object.values(entries).map(stringValue).find(Boolean) ?? null;
|
||||
}
|
||||
|
||||
function relationshipNames(relationships: Array<Record<string, unknown>>): string | null {
|
||||
const names = relationships
|
||||
.filter((entry) => entry.type === "author" || entry.type === "artist")
|
||||
.map((entry) => stringValue(objectValue(entry.attributes).name))
|
||||
.filter((entry): entry is string => Boolean(entry));
|
||||
return names.length ? [...new Set(names)].join(", ") : null;
|
||||
}
|
||||
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | null {
|
||||
if (typeof value === "number") return String(value);
|
||||
return typeof value === "string" && value.trim() ? value.trim() : null;
|
||||
}
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig, MetadataSearchQuery } from "../metadata.types.js";
|
||||
import { toIsbn13 } from "../use-cases/extract-identifiers.js";
|
||||
import { normalizePublishedDate } from "../use-cases/normalize-published-date.js";
|
||||
import { providerFetch, providerHttpError } from "./provider-fetch.js";
|
||||
|
||||
@Injectable()
|
||||
export class OpenLibraryProvider implements MetadataProvider {
|
||||
@ -24,11 +26,11 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
if (query.author) url.searchParams.set("author", query.author);
|
||||
if (query.year) url.searchParams.set("first_publish_year", query.year);
|
||||
url.searchParams.set("limit", "5");
|
||||
const response = await fetch(url, {
|
||||
const response = await providerFetch(this.id, url, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return [];
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
const data = (await response.json()) as { docs?: Array<Record<string, unknown>> };
|
||||
return (data.docs ?? []).map((doc) => ({
|
||||
title: stringValue(doc.title) ?? undefined,
|
||||
@ -36,28 +38,31 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
author: arrayJoin(doc.author_name),
|
||||
language: firstArrayValue(doc.language),
|
||||
publisher: firstArrayValue(doc.publisher),
|
||||
publishedDate: String(doc.first_publish_year ?? "") || null,
|
||||
isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null)
|
||||
publishedDate: normalizePublishedDate(String(doc.first_publish_year ?? "") || null),
|
||||
isbn: bestIsbn(doc.isbn, query.isbn ? toIsbn13(query.isbn) : null),
|
||||
coverUrl: openLibraryCoverUrl(doc.cover_i, firstArrayValue(doc.edition_key) ?? stringValue(doc.cover_edition_key))
|
||||
}));
|
||||
}
|
||||
|
||||
private async lookupEdition(sourceId: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const editionKey = sourceId.replace(/^\/?books\//, "");
|
||||
if (!editionKey) return null;
|
||||
const response = await fetch(`https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org/books/${encodeURIComponent(editionKey)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
private async lookupIsbn(isbn: string, expectedIsbn13: string | null): Promise<MetadataMatch | null> {
|
||||
const response = await fetch(`https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org/isbn/${encodeURIComponent(isbn)}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(4000)
|
||||
timeoutMs: 4000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
return this.editionToMatch((await response.json()) as Record<string, unknown>, expectedIsbn13);
|
||||
}
|
||||
|
||||
@ -70,18 +75,20 @@ export class OpenLibraryProvider implements MetadataProvider {
|
||||
isbn: bestIsbn([...(asStringArray(edition.isbn_13)), ...(asStringArray(edition.isbn_10))], expectedIsbn13),
|
||||
language: languageValue(edition.languages),
|
||||
publisher: firstArrayValue(edition.publishers),
|
||||
publishedDate: stringValue(edition.publish_date)
|
||||
publishedDate: normalizePublishedDate(stringValue(edition.publish_date)),
|
||||
coverUrl: editionCoverUrl(edition)
|
||||
};
|
||||
}
|
||||
|
||||
private async lookupAuthorName(value: unknown): Promise<string | null> {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
if (typeof key !== "string") return null;
|
||||
const response = await fetch(`https://openlibrary.org${key}.json`, {
|
||||
const response = await providerFetch(this.id, `https://openlibrary.org${key}.json`, {
|
||||
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
|
||||
signal: AbortSignal.timeout(3000)
|
||||
timeoutMs: 3000
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw await providerHttpError(this.id, response, `OpenLibrary HTTP ${response.status}`);
|
||||
const author = (await response.json()) as Record<string, unknown>;
|
||||
return stringValue(author.name);
|
||||
}
|
||||
@ -124,3 +131,18 @@ function languageValue(value: unknown): string | null {
|
||||
const key = (Array.isArray(value) ? value[0] : undefined)?.key;
|
||||
return typeof key === "string" ? key.split("/").pop() ?? null : null;
|
||||
}
|
||||
|
||||
function openLibraryCoverUrl(coverId: unknown, editionKey: string | null): string | null {
|
||||
if (typeof coverId === "number" || typeof coverId === "string") {
|
||||
return `https://covers.openlibrary.org/b/id/${encodeURIComponent(String(coverId))}-L.jpg`;
|
||||
}
|
||||
if (editionKey) {
|
||||
return `https://covers.openlibrary.org/b/olid/${encodeURIComponent(editionKey)}-L.jpg`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function editionCoverUrl(edition: Record<string, unknown>): string | null {
|
||||
const covers = Array.isArray(edition.covers) ? edition.covers : [];
|
||||
return openLibraryCoverUrl(covers[0], stringValue(edition.key)?.split("/").pop() ?? null);
|
||||
}
|
||||
|
||||
88
apps/api/src/metadata/adapters/provider-fetch.ts
Normal file
88
apps/api/src/metadata/adapters/provider-fetch.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import { MetadataProviderId } from "../metadata.types.js";
|
||||
|
||||
export type MetadataProviderFailureCode = "timeout" | "dns" | "quota" | "auth" | "http" | "network";
|
||||
|
||||
export class MetadataProviderRequestError extends Error {
|
||||
constructor(
|
||||
readonly provider: MetadataProviderId | "cover",
|
||||
readonly code: MetadataProviderFailureCode,
|
||||
readonly message: string,
|
||||
readonly status?: number
|
||||
) {
|
||||
super(message);
|
||||
this.name = "MetadataProviderRequestError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function providerFetch(
|
||||
provider: MetadataProviderId | "cover",
|
||||
input: string | URL,
|
||||
init: RequestInit & { timeoutMs: number }
|
||||
): Promise<Response> {
|
||||
const { timeoutMs, ...requestInit } = init;
|
||||
try {
|
||||
return await fetch(input, {
|
||||
...requestInit,
|
||||
signal: requestInit.signal ?? AbortSignal.timeout(timeoutMs)
|
||||
});
|
||||
} catch (error) {
|
||||
throw classifyFetchError(provider, error, timeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
export async function providerHttpError(
|
||||
provider: MetadataProviderId | "cover",
|
||||
response: Response,
|
||||
fallbackMessage: string
|
||||
): Promise<MetadataProviderRequestError> {
|
||||
const message = (await response.text().catch(() => "")) || fallbackMessage;
|
||||
if (response.status === 429) return new MetadataProviderRequestError(provider, "quota", message, response.status);
|
||||
if (response.status === 401 || response.status === 403) return new MetadataProviderRequestError(provider, "auth", message, response.status);
|
||||
return new MetadataProviderRequestError(provider, "http", message, response.status);
|
||||
}
|
||||
|
||||
export function describeMetadataProviderError(error: unknown): string {
|
||||
if (error instanceof MetadataProviderRequestError) {
|
||||
const status = error.status ? ` HTTP ${error.status}` : "";
|
||||
return `${error.code}${status}: ${error.message}`;
|
||||
}
|
||||
if (hasProviderErrorCode(error)) {
|
||||
const status = typeof error.status === "number" ? ` HTTP ${error.status}` : "";
|
||||
return `${String(error.code)}${status}: ${errorMessage(error)}`;
|
||||
}
|
||||
return errorMessage(error);
|
||||
}
|
||||
|
||||
function classifyFetchError(provider: MetadataProviderId | "cover", error: unknown, timeoutMs: number): MetadataProviderRequestError {
|
||||
const code = nestedCode(error);
|
||||
if (isTimeoutError(error)) {
|
||||
return new MetadataProviderRequestError(provider, "timeout", `request timed out after ${timeoutMs}ms`);
|
||||
}
|
||||
if (code === "EAI_AGAIN" || code === "ENOTFOUND") {
|
||||
return new MetadataProviderRequestError(provider, "dns", code);
|
||||
}
|
||||
return new MetadataProviderRequestError(provider, "network", errorMessage(error));
|
||||
}
|
||||
|
||||
function isTimeoutError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError") ||
|
||||
error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")
|
||||
);
|
||||
}
|
||||
|
||||
function nestedCode(error: unknown): string | null {
|
||||
if (!error || typeof error !== "object") return null;
|
||||
const direct = "code" in error && typeof error.code === "string" ? error.code : null;
|
||||
if (direct) return direct;
|
||||
const cause = "cause" in error ? error.cause : null;
|
||||
return cause && typeof cause === "object" && "code" in cause && typeof cause.code === "string" ? cause.code : null;
|
||||
}
|
||||
|
||||
function hasProviderErrorCode(error: unknown): error is { code: string; status?: number; message?: string } {
|
||||
return Boolean(error && typeof error === "object" && "code" in error && typeof error.code === "string");
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
Reference in New Issue
Block a user