feat(api,web): providers ComicVine + MangaDex et pilotage de l'enrichissement
- Adaptateurs comic-vine et mangadex avec helper de fetch partagé, timeouts et fallback durcis sur les providers existants - Scoring des correspondances amélioré, normalisation de la date de publication, chaîne de résolution des providers étendue - Scanner : statuts par livre (scan/enrichissement) et page admin d'automatisation alignée Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@ -3,9 +3,11 @@ import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { basename, extname, join } from "node:path";
|
||||
import { eq, inArray } from "drizzle-orm";
|
||||
import { DatabaseService } from "../database/database.service.js";
|
||||
import { automationSettings, books, libraries } from "../database/schema.js";
|
||||
import { automationSettings, books, libraries, series } from "../database/schema.js";
|
||||
import { JobsService } from "../jobs/jobs.service.js";
|
||||
import { MetadataService } from "../metadata/metadata.service.js";
|
||||
import { extractSeriesVolume } from "../metadata/use-cases/extract-series-volume.js";
|
||||
import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js";
|
||||
import { extractMetadata } from "./metadata.js";
|
||||
|
||||
@Injectable()
|
||||
@ -86,23 +88,40 @@ export class ScannerService {
|
||||
this.jobs.markRunning(jobId, "Enriching existing books");
|
||||
const rows = this.database.db.select({ id: books.id }).from(books).all();
|
||||
let count = 0;
|
||||
const failures: ScanFailure[] = [];
|
||||
for (const row of rows) {
|
||||
await this.metadata.enrichBook(row.id);
|
||||
count += 1;
|
||||
this.markBookEnrichmentStatus(row.id, "running");
|
||||
try {
|
||||
await this.metadata.enrichBook(row.id);
|
||||
this.markBookEnrichmentStatus(row.id, "succeeded");
|
||||
count += 1;
|
||||
} catch (error) {
|
||||
this.markBookEnrichmentStatus(row.id, "failed");
|
||||
failures.push({ filePath: `book #${row.id}`, error: errorMessage(error) });
|
||||
}
|
||||
}
|
||||
this.jobs.markSucceeded(jobId, `Enriched ${count} book(s)`);
|
||||
this.jobs.markSucceeded(jobId, enrichmentDigest(count, failures));
|
||||
}
|
||||
|
||||
private async ingestFile(libraryId: number, filePath: string): Promise<void> {
|
||||
const stats = statSync(filePath);
|
||||
const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get();
|
||||
if (existing) {
|
||||
this.database.db
|
||||
.update(books)
|
||||
.set({ scanStatus: "running", enrichmentStatus: "running", updatedAt: this.database.now() })
|
||||
.where(eq(books.id, existing.id))
|
||||
.run();
|
||||
}
|
||||
const localMetadata = await extractMetadata(filePath, this.database.config.storageDir);
|
||||
const now = this.database.now();
|
||||
const format = bookFormatFromPath(filePath);
|
||||
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
|
||||
const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true;
|
||||
const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich });
|
||||
const seriesInfo = this.resolveSeries(metadata.title, filePath);
|
||||
const values = {
|
||||
libraryId,
|
||||
seriesId: seriesInfo.seriesId,
|
||||
title: metadata.title,
|
||||
author: metadata.author,
|
||||
description: metadata.description,
|
||||
@ -113,25 +132,31 @@ export class ScannerService {
|
||||
language: metadata.language,
|
||||
publisher: metadata.publisher,
|
||||
publishedDate: metadata.publishedDate,
|
||||
volumeNumber: seriesInfo.volumeNumber,
|
||||
volumeLabel: seriesInfo.volumeLabel,
|
||||
format,
|
||||
filePath,
|
||||
coverPath: metadata.coverPath,
|
||||
metadataStatus: metadata.metadataStatus,
|
||||
metadataProvenanceJson: metadata.metadataProvenanceJson,
|
||||
scanStatus: "succeeded" as const,
|
||||
enrichmentStatus: shouldRemoteEnrich ? ("succeeded" as const) : ("idle" as const),
|
||||
fileSize: stats.size,
|
||||
fileMtime: stats.mtime.toISOString(),
|
||||
updatedAt: now
|
||||
};
|
||||
|
||||
existing
|
||||
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
|
||||
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
|
||||
this.upsertBookByFilePath(values, existing, now);
|
||||
}
|
||||
|
||||
private ingestIncompleteFile(libraryId: number, filePath: string): void {
|
||||
const stats = statSync(filePath);
|
||||
const now = this.database.now();
|
||||
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
|
||||
const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get();
|
||||
const seriesInfo = this.resolveSeries(basename(filePath, extname(filePath)), filePath);
|
||||
const values = {
|
||||
libraryId,
|
||||
seriesId: seriesInfo.seriesId,
|
||||
title: basename(filePath, extname(filePath)),
|
||||
author: null,
|
||||
description: null,
|
||||
@ -154,17 +179,41 @@ export class ScannerService {
|
||||
language: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
volumeNumber: seriesInfo.volumeNumber,
|
||||
volumeLabel: seriesInfo.volumeLabel,
|
||||
format: bookFormatFromPath(filePath),
|
||||
filePath,
|
||||
coverPath: null,
|
||||
metadataStatus: "none" as const,
|
||||
metadataProvenanceJson: JSON.stringify({ title: "local" }),
|
||||
scanStatus: "failed" as const,
|
||||
enrichmentStatus: "failed" as const,
|
||||
fileSize: stats.size,
|
||||
fileMtime: stats.mtime.toISOString(),
|
||||
updatedAt: now
|
||||
};
|
||||
|
||||
existing
|
||||
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
|
||||
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
|
||||
this.upsertBookByFilePath(values, existing, now);
|
||||
}
|
||||
|
||||
private upsertBookByFilePath(
|
||||
values: Omit<typeof books.$inferInsert, "createdAt">,
|
||||
existing: typeof books.$inferSelect | undefined,
|
||||
createdAt: string
|
||||
): void {
|
||||
if (existing) {
|
||||
this.database.db.update(books).set(preserveExistingBookValues(values, existing)).where(eq(books.id, existing.id)).run();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.database.db.insert(books).values({ ...values, createdAt }).run();
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isUniqueFilePathError(error)) throw error;
|
||||
const current = this.database.db.select().from(books).where(eq(books.filePath, values.filePath)).get();
|
||||
if (!current) throw error;
|
||||
this.database.db.update(books).set(preserveExistingBookValues(values, current)).where(eq(books.id, current.id)).run();
|
||||
}
|
||||
}
|
||||
|
||||
private removeMissingBooks(libraryId: number, seen: Set<string>): number {
|
||||
@ -184,6 +233,32 @@ export class ScannerService {
|
||||
.get()?.autoEnrichNewBooks
|
||||
);
|
||||
}
|
||||
|
||||
private markBookEnrichmentStatus(id: number, enrichmentStatus: "running" | "succeeded" | "failed"): void {
|
||||
this.database.db.update(books).set({ enrichmentStatus, updatedAt: this.database.now() }).where(eq(books.id, id)).run();
|
||||
}
|
||||
|
||||
private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } {
|
||||
const parsed = extractSeriesVolume(title, filePath);
|
||||
const now = this.database.now();
|
||||
const row = this.database.db
|
||||
.insert(series)
|
||||
.values({
|
||||
title: parsed.seriesTitle,
|
||||
normalizedTitle: parsed.normalizedSeriesTitle,
|
||||
description: null,
|
||||
publisher: null,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: series.normalizedTitle,
|
||||
set: { title: parsed.seriesTitle, updatedAt: now }
|
||||
})
|
||||
.returning({ id: series.id })
|
||||
.get();
|
||||
return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel };
|
||||
}
|
||||
}
|
||||
|
||||
type ScanFailure = {
|
||||
@ -202,6 +277,17 @@ export function scanDigest(scanned: number, removed: number, failures: ScanFailu
|
||||
return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`;
|
||||
}
|
||||
|
||||
export function enrichmentDigest(enriched: number, failures: ScanFailure[]): string {
|
||||
const base = `Enriched ${enriched} book(s)`;
|
||||
if (!failures.length) return base;
|
||||
const examples = failures
|
||||
.slice(0, 3)
|
||||
.map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`)
|
||||
.join("; ");
|
||||
const extra = failures.length > 3 ? `; ${failures.length - 3} more` : "";
|
||||
return `${base}, ${failures.length} incomplete book(s): ${examples}${extra}`;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error && error.message) return truncate(error.message);
|
||||
return truncate(String(error));
|
||||
@ -233,3 +319,49 @@ function bookFormatFromPath(filePath: string): "epub" | "pdf" | "cbz" | "cbr" {
|
||||
if (extension === ".cbr") return "cbr";
|
||||
return "pdf";
|
||||
}
|
||||
|
||||
export function preserveExistingBookValues<T extends Partial<typeof books.$inferInsert>>(values: T, existing: typeof books.$inferSelect): T {
|
||||
const next = { ...values };
|
||||
if (next.scanStatus === "failed" && existing.title) {
|
||||
next.title = existing.title as never;
|
||||
}
|
||||
for (const field of ["author", "description", "isbn", "isbn13", "language", "publisher", "publishedDate", "coverPath", "seriesId", "volumeNumber", "volumeLabel"] as const) {
|
||||
if (field === "publishedDate") {
|
||||
next.publishedDate = (normalizePublishedDate(next.publishedDate) ?? normalizePublishedDate(existing.publishedDate)) as never;
|
||||
continue;
|
||||
}
|
||||
if (next[field] == null && existing[field] != null) {
|
||||
next[field] = existing[field] as never;
|
||||
}
|
||||
}
|
||||
next.metadataStatus = computeMetadataStatus(next, existing.metadataStatus) as never;
|
||||
next.metadataProvenanceJson = mergeProvenanceJson(String(next.metadataProvenanceJson ?? "{}"), existing.metadataProvenanceJson) as never;
|
||||
return next;
|
||||
}
|
||||
|
||||
function computeMetadataStatus(values: Partial<typeof books.$inferInsert>, existingStatus: string): "enriched" | "partial" | "none" {
|
||||
const hasCover = Boolean(values.coverPath);
|
||||
const filled = [values.author, values.description, values.isbn, values.language, values.publisher, values.publishedDate].filter(Boolean).length;
|
||||
const computed = hasCover && filled >= 2 ? "enriched" : hasCover || filled > 0 ? "partial" : "none";
|
||||
const rank = { none: 0, partial: 1, enriched: 2 } as const;
|
||||
const safeExisting = existingStatus === "enriched" || existingStatus === "partial" || existingStatus === "none" ? existingStatus : "none";
|
||||
return rank[computed] >= rank[safeExisting] ? computed : safeExisting;
|
||||
}
|
||||
|
||||
function mergeProvenanceJson(nextJson: string, existingJson: string | null): string {
|
||||
return JSON.stringify({ ...parseJsonObject(existingJson), ...parseJsonObject(nextJson) });
|
||||
}
|
||||
|
||||
function parseJsonObject(value: string | null): Record<string, unknown> {
|
||||
if (!value) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function isUniqueFilePathError(error: unknown): boolean {
|
||||
return error instanceof Error && error.message.includes("UNIQUE constraint failed: books.file_path");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user