feat(api): digest de scan avec fichiers incomplets en cas d'échec
Un fichier dont l'ingestion échoue est enregistré en entrée incomplète (métadonnées minimales) plutôt que d'interrompre le scan; le digest du job liste les échecs avec exemples tronqués. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
20
apps/api/src/scanner/scanner.service.test.ts
Normal file
20
apps/api/src/scanner/scanner.service.test.ts
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { scanDigest } from "./scanner.service.js";
|
||||||
|
|
||||||
|
describe("scan digest", () => {
|
||||||
|
it("reports incomplete files without exposing huge traces", () => {
|
||||||
|
const detail = scanDigest(
|
||||||
|
19,
|
||||||
|
0,
|
||||||
|
[
|
||||||
|
{ filePath: "/library/broken.cbr", error: "x".repeat(300) },
|
||||||
|
{ filePath: "/library/broken.epub", error: "Invalid EPUB" }
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(detail).toContain("Scanned 19 file(s)");
|
||||||
|
expect(detail).toContain("2 incomplete file(s)");
|
||||||
|
expect(detail).toContain("broken.cbr");
|
||||||
|
expect(detail.length).toBeLessThan(380);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -1,6 +1,6 @@
|
|||||||
import { Injectable, NotFoundException } from "@nestjs/common";
|
import { Injectable, NotFoundException } from "@nestjs/common";
|
||||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||||
import { extname, join } from "node:path";
|
import { basename, extname, join } from "node:path";
|
||||||
import { eq, inArray } from "drizzle-orm";
|
import { eq, inArray } from "drizzle-orm";
|
||||||
import { DatabaseService } from "../database/database.service.js";
|
import { DatabaseService } from "../database/database.service.js";
|
||||||
import { automationSettings, books, libraries } from "../database/schema.js";
|
import { automationSettings, books, libraries } from "../database/schema.js";
|
||||||
@ -47,27 +47,39 @@ export class ScannerService {
|
|||||||
private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise<void> {
|
private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise<void> {
|
||||||
this.jobs.markRunning(jobId, `Scanning ${library.path}`);
|
this.jobs.markRunning(jobId, `Scanning ${library.path}`);
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
const failures: ScanFailure[] = [];
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
for (const filePath of walkBooks(library.path)) {
|
for (const filePath of walkBooks(library.path)) {
|
||||||
seen.add(filePath);
|
seen.add(filePath);
|
||||||
await this.ingestFile(library.id, filePath);
|
try {
|
||||||
count += 1;
|
await this.ingestFile(library.id, filePath);
|
||||||
|
count += 1;
|
||||||
|
} catch (error) {
|
||||||
|
failures.push({ filePath, error: errorMessage(error) });
|
||||||
|
this.ingestIncompleteFile(library.id, filePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const removed = this.removeMissingBooks(library.id, seen);
|
const removed = this.removeMissingBooks(library.id, seen);
|
||||||
this.jobs.markSucceeded(jobId, `Scanned ${count} file(s), removed ${removed} missing book(s)`);
|
this.jobs.markSucceeded(jobId, scanDigest(count, removed, failures));
|
||||||
}
|
}
|
||||||
|
|
||||||
private async scanAllLibraries(jobId: number): Promise<void> {
|
private async scanAllLibraries(jobId: number): Promise<void> {
|
||||||
this.jobs.markRunning(jobId, "Scanning all enabled libraries");
|
this.jobs.markRunning(jobId, "Scanning all enabled libraries");
|
||||||
const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all();
|
const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all();
|
||||||
let scanned = 0;
|
let scanned = 0;
|
||||||
|
const failures: ScanFailure[] = [];
|
||||||
for (const library of enabledLibraries) {
|
for (const library of enabledLibraries) {
|
||||||
for (const filePath of walkBooks(library.path)) {
|
for (const filePath of walkBooks(library.path)) {
|
||||||
await this.ingestFile(library.id, filePath);
|
try {
|
||||||
scanned += 1;
|
await this.ingestFile(library.id, filePath);
|
||||||
|
scanned += 1;
|
||||||
|
} catch (error) {
|
||||||
|
failures.push({ filePath, error: errorMessage(error) });
|
||||||
|
this.ingestIncompleteFile(library.id, filePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.jobs.markSucceeded(jobId, `Scanned ${scanned} file(s) across ${enabledLibraries.length} library/libraries`);
|
this.jobs.markSucceeded(jobId, scanDigest(scanned, 0, failures, `across ${enabledLibraries.length} library/libraries`));
|
||||||
}
|
}
|
||||||
|
|
||||||
private async enrichExistingBooks(jobId: number): Promise<void> {
|
private async enrichExistingBooks(jobId: number): Promise<void> {
|
||||||
@ -113,6 +125,34 @@ export class ScannerService {
|
|||||||
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
|
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ingestIncompleteFile(libraryId: number, filePath: string): void {
|
||||||
|
const stats = statSync(filePath);
|
||||||
|
const now = this.database.now();
|
||||||
|
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
|
||||||
|
const values = {
|
||||||
|
libraryId,
|
||||||
|
title: basename(filePath, extname(filePath)),
|
||||||
|
author: null,
|
||||||
|
description: null,
|
||||||
|
isbn: null,
|
||||||
|
isbn13: null,
|
||||||
|
identifiersJson: JSON.stringify({ candidates: [], isbn10: null, isbn13: null }),
|
||||||
|
language: null,
|
||||||
|
publisher: null,
|
||||||
|
publishedDate: null,
|
||||||
|
format: bookFormatFromPath(filePath),
|
||||||
|
filePath,
|
||||||
|
coverPath: null,
|
||||||
|
fileSize: stats.size,
|
||||||
|
fileMtime: stats.mtime.toISOString(),
|
||||||
|
updatedAt: now
|
||||||
|
};
|
||||||
|
|
||||||
|
existing
|
||||||
|
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
|
||||||
|
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
|
||||||
|
}
|
||||||
|
|
||||||
private removeMissingBooks(libraryId: number, seen: Set<string>): number {
|
private removeMissingBooks(libraryId: number, seen: Set<string>): number {
|
||||||
const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all();
|
const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all();
|
||||||
const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath));
|
const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath));
|
||||||
@ -132,6 +172,31 @@ export class ScannerService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ScanFailure = {
|
||||||
|
filePath: string;
|
||||||
|
error: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function scanDigest(scanned: number, removed: number, failures: ScanFailure[], suffix?: string): string {
|
||||||
|
const base = suffix ? `Scanned ${scanned} file(s) ${suffix}` : `Scanned ${scanned} file(s), removed ${removed} missing book(s)`;
|
||||||
|
if (!failures.length) return base;
|
||||||
|
const examples = failures
|
||||||
|
.slice(0, 3)
|
||||||
|
.map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`)
|
||||||
|
.join("; ");
|
||||||
|
const extra = failures.length > 3 ? `; ${failures.length - 3} more` : "";
|
||||||
|
return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
if (error instanceof Error && error.message) return truncate(error.message);
|
||||||
|
return truncate(String(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncate(value: string): string {
|
||||||
|
return value.length > 120 ? `${value.slice(0, 117)}...` : value;
|
||||||
|
}
|
||||||
|
|
||||||
function* walkBooks(root: string): Generator<string> {
|
function* walkBooks(root: string): Generator<string> {
|
||||||
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
||||||
const path = join(root, entry.name);
|
const path = join(root, entry.name);
|
||||||
|
|||||||
Reference in New Issue
Block a user