feat(api,web): providers ComicVine + MangaDex et pilotage de l'enrichissement

- Adaptateurs comic-vine et mangadex avec helper de fetch partagé,
  timeouts et fallback durcis sur les providers existants
- Scoring des correspondances amélioré, normalisation de la date de
  publication, chaîne de résolution des providers étendue
- Scanner : statuts par livre (scan/enrichissement) et page admin
  d'automatisation alignée

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Git Agent
2026-08-24 09:11:18 +02:00
parent 93bb40a8cd
commit d79f502dd2
25 changed files with 2812 additions and 231 deletions

View File

@ -2,10 +2,15 @@ import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import AdmZip from "adm-zip";
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { listCbzImageEntries } from "../common/cbz.js";
import { extractMetadata } from "./metadata.js";
vi.mock("../common/cbr.js", () => ({
listCbrImageEntries: async () => [{ entryName: "001.jpg", name: "001.jpg" }],
readCbrPage: async () => ({ entryName: "001.jpg", data: Buffer.from([0xff, 0xd8, 0xff, 0xd9]) })
}));
describe("pdf metadata extraction", () => {
it("falls back to file name and reads simple PDF info fields", async () => {
const dir = mkdtempSync(join(tmpdir(), "readabook-"));
@ -36,3 +41,16 @@ describe("cbz metadata extraction", () => {
expect(pages.map((page) => page.name)).toEqual(["001.jpg", "002.jpg"]);
});
});
describe("cbr metadata extraction", () => {
it("uses the file name as title and first extracted image as cover", async () => {
const dir = mkdtempSync(join(tmpdir(), "readabook-"));
const file = join(dir, "Comic Two.cbr");
writeFileSync(file, "rar");
const metadata = await extractMetadata(file, dir);
expect(metadata.title).toBe("Comic Two");
expect(metadata.coverPath).toMatch(/covers\/[a-f0-9]+\.jpg$/);
});
});

View File

@ -3,8 +3,9 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { basename, dirname, extname, join } from "node:path";
import AdmZip from "adm-zip";
import { XMLParser } from "fast-xml-parser";
import { listCbrImageEntries } from "../common/cbr.js";
import { listCbrImageEntries, readCbrPage } from "../common/cbr.js";
import { listCbzImageEntries } from "../common/cbz.js";
import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js";
export type BookMetadata = {
title: string;
@ -64,7 +65,7 @@ function extractEpubMetadata(filePath: string, storageDir: string): BookMetadata
isbn,
language: firstText(metadata["dc:language"]),
publisher: firstText(metadata["dc:publisher"]),
publishedDate: firstText(metadata["dc:date"]),
publishedDate: normalizePublishedDate(firstText(metadata["dc:date"])),
coverPath
};
}
@ -97,10 +98,12 @@ function extractCbzMetadata(filePath: string, storageDir: string): BookMetadata
}
async function extractCbrMetadata(filePath: string, storageDir: string): Promise<BookMetadata> {
await listCbrImageEntries(filePath);
const firstPage = (await listCbrImageEntries(filePath))[0];
const page = await readCbrPage(filePath, 1, storageDir);
const coverPath = writeCoverData(page.data, firstPage.entryName, filePath, storageDir);
return {
...fallbackMetadata(filePath),
coverPath: null
coverPath
};
}
@ -165,6 +168,15 @@ function extractCover(zip: AdmZip, coverPathInZip: string, filePath: string, sto
return target;
}
function writeCoverData(data: Buffer, entryName: string, filePath: string, storageDir: string): string {
const extension = extname(entryName) || ".jpg";
const hash = createHash("sha256").update(filePath).digest("hex").slice(0, 24);
const target = join(storageDir, "covers", `${hash}${extension}`);
mkdirSync(dirname(target), { recursive: true });
writeFileSync(target, data);
return target;
}
function matchPdfInfo(text: string, key: string): string | null {
return text.match(new RegExp(`/${key}\\s*\\(([^)]{1,500})\\)`))?.[1] ?? null;
}

View File

@ -1,5 +1,24 @@
import { describe, expect, it } from "vitest";
import { scanDigest } from "./scanner.service.js";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { DatabaseService } from "../database/database.service.js";
import { books, libraries } from "../database/schema.js";
import { JobsService } from "../jobs/jobs.service.js";
import { enrichmentDigest, preserveExistingBookValues, scanDigest } from "./scanner.service.js";
import { ScannerService } from "./scanner.service.js";
const previousDatabasePath = process.env.DATABASE_PATH;
const previousStorageDir = process.env.STORAGE_DIR;
const tempDirs: string[] = [];
afterEach(() => {
process.env.DATABASE_PATH = previousDatabasePath;
process.env.STORAGE_DIR = previousStorageDir;
for (const dir of tempDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
});
describe("scan digest", () => {
it("reports incomplete files without exposing huge traces", () => {
@ -17,4 +36,217 @@ describe("scan digest", () => {
expect(detail).toContain("broken.cbr");
expect(detail.length).toBeLessThan(380);
});
it("reports metadata enrichment jobs as enrichment, not scans", () => {
expect(enrichmentDigest(35, [])).toBe("Enriched 35 book(s)");
});
it("does not erase existing metadata or cover when a rescan has less information", () => {
const existing: typeof books.$inferSelect = {
id: 1,
libraryId: 1,
seriesId: null,
title: "Daredevil",
author: "Roy Thomas",
description: "Existing description",
isbn: "9782809476255",
isbn13: "9782809476255",
identifiersJson: null,
localMetadataJson: null,
language: "fre",
publisher: "Panini comics",
publishedDate: "2019",
volumeNumber: null,
volumeLabel: null,
format: "cbz",
filePath: "/library/Daredevil.cbz",
coverPath: "/storage/covers/daredevil.jpg",
metadataStatus: "enriched",
metadataProvenanceJson: JSON.stringify({ author: "bnf", coverPath: "openlibrary" }),
scanStatus: "succeeded",
enrichmentStatus: "succeeded",
fileSize: 12,
fileMtime: "2026-08-23T00:00:00.000Z",
createdAt: "2026-08-23T00:00:00.000Z",
updatedAt: "2026-08-23T00:00:00.000Z"
};
const next = preserveExistingBookValues(
{
title: "Daredevil",
author: null,
description: null,
isbn: null,
isbn13: null,
language: null,
publisher: null,
publishedDate: null,
coverPath: null,
metadataStatus: "none",
metadataProvenanceJson: JSON.stringify({ title: "local" }),
scanStatus: "succeeded" as const
},
existing
);
expect(next).toMatchObject({
author: "Roy Thomas",
description: "Existing description",
isbn: "9782809476255",
coverPath: "/storage/covers/daredevil.jpg",
metadataStatus: "enriched"
});
expect(JSON.parse(String(next.metadataProvenanceJson))).toMatchObject({
title: "local",
author: "bnf",
coverPath: "openlibrary"
});
});
it("does not replace an existing valid publication date with a sentinel date", () => {
const existing = {
id: 1,
libraryId: 1,
seriesId: null,
title: "Lord of the Mysteries",
author: null,
description: null,
isbn: null,
isbn13: null,
identifiersJson: null,
localMetadataJson: null,
language: null,
publisher: null,
publishedDate: "2018",
volumeNumber: null,
volumeLabel: null,
format: "epub",
filePath: "/library/Lord of the Mysteries.epub",
coverPath: null,
metadataStatus: "partial",
metadataProvenanceJson: JSON.stringify({ publishedDate: "existing" }),
scanStatus: "succeeded",
enrichmentStatus: "succeeded",
fileSize: 12,
fileMtime: "2026-08-23T00:00:00.000Z",
createdAt: "2026-08-23T00:00:00.000Z",
updatedAt: "2026-08-23T00:00:00.000Z"
} satisfies typeof books.$inferSelect;
const next = preserveExistingBookValues(
{
title: "Lord of the Mysteries",
publishedDate: "0101-01-01T00:00:00+00:00",
metadataStatus: "partial",
metadataProvenanceJson: JSON.stringify({ title: "local", publishedDate: "openlibrary" })
},
existing
);
expect(next.publishedDate).toBe("2018");
});
it.runIf(canLoadBetterSqlite())("updates the existing book when an insert races with books.file_path uniqueness", () => {
const database = createDatabase();
const now = database.now();
const library = database.db
.insert(libraries)
.values({ name: "Corpus", path: "/library", enabled: true, createdAt: now, updatedAt: now })
.returning()
.get();
database.db
.insert(books)
.values({
libraryId: library.id,
seriesId: null,
title: "Daredevil",
author: null,
description: null,
isbn: null,
isbn13: null,
identifiersJson: null,
localMetadataJson: null,
language: null,
publisher: null,
publishedDate: null,
volumeNumber: null,
volumeLabel: null,
format: "cbz",
filePath: "/library/Daredevil.cbz",
coverPath: null,
metadataStatus: "none",
metadataProvenanceJson: null,
scanStatus: "succeeded",
enrichmentStatus: "succeeded",
fileSize: 1,
fileMtime: now,
createdAt: now,
updatedAt: now
})
.run();
const scanner = new ScannerService(database, new JobsService(database), {} as never);
const values: Omit<typeof books.$inferInsert, "createdAt"> = {
libraryId: library.id,
seriesId: null,
title: "Daredevil",
author: "Roy Thomas",
description: "Updated metadata",
isbn: null,
isbn13: null,
identifiersJson: null,
localMetadataJson: null,
language: null,
publisher: null,
publishedDate: null,
volumeNumber: null,
volumeLabel: null,
format: "cbz",
filePath: "/library/Daredevil.cbz",
coverPath: "/storage/covers/daredevil.jpg",
metadataStatus: "enriched",
metadataProvenanceJson: JSON.stringify({ author: "bnf", coverPath: "local" }),
scanStatus: "succeeded",
enrichmentStatus: "succeeded",
fileSize: 2,
fileMtime: now,
updatedAt: now
};
(scanner as unknown as {
upsertBookByFilePath(values: Omit<typeof books.$inferInsert, "createdAt">, existing: undefined, createdAt: string): void;
}).upsertBookByFilePath(
values,
undefined,
now
);
const rows = database.db.select().from(books).all();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({
author: "Roy Thomas",
coverPath: "/storage/covers/daredevil.jpg",
metadataStatus: "enriched",
fileSize: 2
});
database.onModuleDestroy();
});
});
function createDatabase(): DatabaseService {
const dir = mkdtempSync(join(tmpdir(), "readabook-scanner-service-"));
tempDirs.push(dir);
process.env.DATABASE_PATH = join(dir, "readabook.sqlite");
process.env.STORAGE_DIR = join(dir, "storage");
return new DatabaseService();
}
function canLoadBetterSqlite(): boolean {
try {
const database = createDatabase();
database.onModuleDestroy();
return true;
} catch {
return false;
}
}

View File

@ -3,9 +3,11 @@ import { existsSync, readdirSync, statSync } from "node:fs";
import { basename, extname, join } from "node:path";
import { eq, inArray } from "drizzle-orm";
import { DatabaseService } from "../database/database.service.js";
import { automationSettings, books, libraries } from "../database/schema.js";
import { automationSettings, books, libraries, series } from "../database/schema.js";
import { JobsService } from "../jobs/jobs.service.js";
import { MetadataService } from "../metadata/metadata.service.js";
import { extractSeriesVolume } from "../metadata/use-cases/extract-series-volume.js";
import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js";
import { extractMetadata } from "./metadata.js";
@Injectable()
@ -86,23 +88,40 @@ export class ScannerService {
this.jobs.markRunning(jobId, "Enriching existing books");
const rows = this.database.db.select({ id: books.id }).from(books).all();
let count = 0;
const failures: ScanFailure[] = [];
for (const row of rows) {
await this.metadata.enrichBook(row.id);
count += 1;
this.markBookEnrichmentStatus(row.id, "running");
try {
await this.metadata.enrichBook(row.id);
this.markBookEnrichmentStatus(row.id, "succeeded");
count += 1;
} catch (error) {
this.markBookEnrichmentStatus(row.id, "failed");
failures.push({ filePath: `book #${row.id}`, error: errorMessage(error) });
}
}
this.jobs.markSucceeded(jobId, `Enriched ${count} book(s)`);
this.jobs.markSucceeded(jobId, enrichmentDigest(count, failures));
}
private async ingestFile(libraryId: number, filePath: string): Promise<void> {
const stats = statSync(filePath);
const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get();
if (existing) {
this.database.db
.update(books)
.set({ scanStatus: "running", enrichmentStatus: "running", updatedAt: this.database.now() })
.where(eq(books.id, existing.id))
.run();
}
const localMetadata = await extractMetadata(filePath, this.database.config.storageDir);
const now = this.database.now();
const format = bookFormatFromPath(filePath);
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true;
const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich });
const seriesInfo = this.resolveSeries(metadata.title, filePath);
const values = {
libraryId,
seriesId: seriesInfo.seriesId,
title: metadata.title,
author: metadata.author,
description: metadata.description,
@ -113,25 +132,31 @@ export class ScannerService {
language: metadata.language,
publisher: metadata.publisher,
publishedDate: metadata.publishedDate,
volumeNumber: seriesInfo.volumeNumber,
volumeLabel: seriesInfo.volumeLabel,
format,
filePath,
coverPath: metadata.coverPath,
metadataStatus: metadata.metadataStatus,
metadataProvenanceJson: metadata.metadataProvenanceJson,
scanStatus: "succeeded" as const,
enrichmentStatus: shouldRemoteEnrich ? ("succeeded" as const) : ("idle" as const),
fileSize: stats.size,
fileMtime: stats.mtime.toISOString(),
updatedAt: now
};
existing
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
this.upsertBookByFilePath(values, existing, now);
}
private ingestIncompleteFile(libraryId: number, filePath: string): void {
const stats = statSync(filePath);
const now = this.database.now();
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get();
const seriesInfo = this.resolveSeries(basename(filePath, extname(filePath)), filePath);
const values = {
libraryId,
seriesId: seriesInfo.seriesId,
title: basename(filePath, extname(filePath)),
author: null,
description: null,
@ -154,17 +179,41 @@ export class ScannerService {
language: null,
publisher: null,
publishedDate: null,
volumeNumber: seriesInfo.volumeNumber,
volumeLabel: seriesInfo.volumeLabel,
format: bookFormatFromPath(filePath),
filePath,
coverPath: null,
metadataStatus: "none" as const,
metadataProvenanceJson: JSON.stringify({ title: "local" }),
scanStatus: "failed" as const,
enrichmentStatus: "failed" as const,
fileSize: stats.size,
fileMtime: stats.mtime.toISOString(),
updatedAt: now
};
existing
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
this.upsertBookByFilePath(values, existing, now);
}
private upsertBookByFilePath(
values: Omit<typeof books.$inferInsert, "createdAt">,
existing: typeof books.$inferSelect | undefined,
createdAt: string
): void {
if (existing) {
this.database.db.update(books).set(preserveExistingBookValues(values, existing)).where(eq(books.id, existing.id)).run();
return;
}
try {
this.database.db.insert(books).values({ ...values, createdAt }).run();
return;
} catch (error) {
if (!isUniqueFilePathError(error)) throw error;
const current = this.database.db.select().from(books).where(eq(books.filePath, values.filePath)).get();
if (!current) throw error;
this.database.db.update(books).set(preserveExistingBookValues(values, current)).where(eq(books.id, current.id)).run();
}
}
private removeMissingBooks(libraryId: number, seen: Set<string>): number {
@ -184,6 +233,32 @@ export class ScannerService {
.get()?.autoEnrichNewBooks
);
}
private markBookEnrichmentStatus(id: number, enrichmentStatus: "running" | "succeeded" | "failed"): void {
this.database.db.update(books).set({ enrichmentStatus, updatedAt: this.database.now() }).where(eq(books.id, id)).run();
}
private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } {
const parsed = extractSeriesVolume(title, filePath);
const now = this.database.now();
const row = this.database.db
.insert(series)
.values({
title: parsed.seriesTitle,
normalizedTitle: parsed.normalizedSeriesTitle,
description: null,
publisher: null,
createdAt: now,
updatedAt: now
})
.onConflictDoUpdate({
target: series.normalizedTitle,
set: { title: parsed.seriesTitle, updatedAt: now }
})
.returning({ id: series.id })
.get();
return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel };
}
}
type ScanFailure = {
@ -202,6 +277,17 @@ export function scanDigest(scanned: number, removed: number, failures: ScanFailu
return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`;
}
export function enrichmentDigest(enriched: number, failures: ScanFailure[]): string {
const base = `Enriched ${enriched} book(s)`;
if (!failures.length) return base;
const examples = failures
.slice(0, 3)
.map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`)
.join("; ");
const extra = failures.length > 3 ? `; ${failures.length - 3} more` : "";
return `${base}, ${failures.length} incomplete book(s): ${examples}${extra}`;
}
function errorMessage(error: unknown): string {
if (error instanceof Error && error.message) return truncate(error.message);
return truncate(String(error));
@ -233,3 +319,49 @@ function bookFormatFromPath(filePath: string): "epub" | "pdf" | "cbz" | "cbr" {
if (extension === ".cbr") return "cbr";
return "pdf";
}
export function preserveExistingBookValues<T extends Partial<typeof books.$inferInsert>>(values: T, existing: typeof books.$inferSelect): T {
const next = { ...values };
if (next.scanStatus === "failed" && existing.title) {
next.title = existing.title as never;
}
for (const field of ["author", "description", "isbn", "isbn13", "language", "publisher", "publishedDate", "coverPath", "seriesId", "volumeNumber", "volumeLabel"] as const) {
if (field === "publishedDate") {
next.publishedDate = (normalizePublishedDate(next.publishedDate) ?? normalizePublishedDate(existing.publishedDate)) as never;
continue;
}
if (next[field] == null && existing[field] != null) {
next[field] = existing[field] as never;
}
}
next.metadataStatus = computeMetadataStatus(next, existing.metadataStatus) as never;
next.metadataProvenanceJson = mergeProvenanceJson(String(next.metadataProvenanceJson ?? "{}"), existing.metadataProvenanceJson) as never;
return next;
}
function computeMetadataStatus(values: Partial<typeof books.$inferInsert>, existingStatus: string): "enriched" | "partial" | "none" {
const hasCover = Boolean(values.coverPath);
const filled = [values.author, values.description, values.isbn, values.language, values.publisher, values.publishedDate].filter(Boolean).length;
const computed = hasCover && filled >= 2 ? "enriched" : hasCover || filled > 0 ? "partial" : "none";
const rank = { none: 0, partial: 1, enriched: 2 } as const;
const safeExisting = existingStatus === "enriched" || existingStatus === "partial" || existingStatus === "none" ? existingStatus : "none";
return rank[computed] >= rank[safeExisting] ? computed : safeExisting;
}
function mergeProvenanceJson(nextJson: string, existingJson: string | null): string {
return JSON.stringify({ ...parseJsonObject(existingJson), ...parseJsonObject(nextJson) });
}
function parseJsonObject(value: string | null): Record<string, unknown> {
if (!value) return {};
try {
const parsed = JSON.parse(value) as unknown;
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
} catch {
return {};
}
}
function isUniqueFilePathError(error: unknown): boolean {
return error instanceof Error && error.message.includes("UNIQUE constraint failed: books.file_path");
}