diff --git a/apps/api/src/scanner/scanner.service.test.ts b/apps/api/src/scanner/scanner.service.test.ts new file mode 100644 index 0000000..2238eee --- /dev/null +++ b/apps/api/src/scanner/scanner.service.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { scanDigest } from "./scanner.service.js"; + +describe("scan digest", () => { + it("reports incomplete files without exposing huge traces", () => { + const detail = scanDigest( + 19, + 0, + [ + { filePath: "/library/broken.cbr", error: "x".repeat(300) }, + { filePath: "/library/broken.epub", error: "Invalid EPUB" } + ] + ); + + expect(detail).toContain("Scanned 19 file(s)"); + expect(detail).toContain("2 incomplete file(s)"); + expect(detail).toContain("broken.cbr"); + expect(detail.length).toBeLessThan(380); + }); +}); diff --git a/apps/api/src/scanner/scanner.service.ts b/apps/api/src/scanner/scanner.service.ts index d6ad412..e8aaeaf 100644 --- a/apps/api/src/scanner/scanner.service.ts +++ b/apps/api/src/scanner/scanner.service.ts @@ -1,6 +1,6 @@ import { Injectable, NotFoundException } from "@nestjs/common"; import { existsSync, readdirSync, statSync } from "node:fs"; -import { extname, join } from "node:path"; +import { basename, extname, join } from "node:path"; import { eq, inArray } from "drizzle-orm"; import { DatabaseService } from "../database/database.service.js"; import { automationSettings, books, libraries } from "../database/schema.js"; @@ -47,27 +47,39 @@ export class ScannerService { private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise { this.jobs.markRunning(jobId, `Scanning ${library.path}`); let count = 0; + const failures: ScanFailure[] = []; const seen = new Set(); for (const filePath of walkBooks(library.path)) { seen.add(filePath); - await this.ingestFile(library.id, filePath); - count += 1; + try { + await this.ingestFile(library.id, filePath); + count += 1; + } catch (error) { + failures.push({ filePath, error: errorMessage(error) }); + this.ingestIncompleteFile(library.id, filePath); + } } const removed = this.removeMissingBooks(library.id, seen); - this.jobs.markSucceeded(jobId, `Scanned ${count} file(s), removed ${removed} missing book(s)`); + this.jobs.markSucceeded(jobId, scanDigest(count, removed, failures)); } private async scanAllLibraries(jobId: number): Promise { this.jobs.markRunning(jobId, "Scanning all enabled libraries"); const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all(); let scanned = 0; + const failures: ScanFailure[] = []; for (const library of enabledLibraries) { for (const filePath of walkBooks(library.path)) { - await this.ingestFile(library.id, filePath); - scanned += 1; + try { + await this.ingestFile(library.id, filePath); + scanned += 1; + } catch (error) { + failures.push({ filePath, error: errorMessage(error) }); + this.ingestIncompleteFile(library.id, filePath); + } } } - this.jobs.markSucceeded(jobId, `Scanned ${scanned} file(s) across ${enabledLibraries.length} library/libraries`); + this.jobs.markSucceeded(jobId, scanDigest(scanned, 0, failures, `across ${enabledLibraries.length} library/libraries`)); } private async enrichExistingBooks(jobId: number): Promise { @@ -113,6 +125,34 @@ export class ScannerService { : this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get(); } + private ingestIncompleteFile(libraryId: number, filePath: string): void { + const stats = statSync(filePath); + const now = this.database.now(); + const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get(); + const values = { + libraryId, + title: basename(filePath, extname(filePath)), + author: null, + description: null, + isbn: null, + isbn13: null, + identifiersJson: JSON.stringify({ candidates: [], isbn10: null, isbn13: null }), + language: null, + publisher: null, + publishedDate: null, + format: bookFormatFromPath(filePath), + filePath, + coverPath: null, + fileSize: stats.size, + fileMtime: stats.mtime.toISOString(), + updatedAt: now + }; + + existing + ? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get() + : this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get(); + } + private removeMissingBooks(libraryId: number, seen: Set): number { const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all(); const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath)); @@ -132,6 +172,31 @@ export class ScannerService { } } +type ScanFailure = { + filePath: string; + error: string; +}; + +export function scanDigest(scanned: number, removed: number, failures: ScanFailure[], suffix?: string): string { + const base = suffix ? `Scanned ${scanned} file(s) ${suffix}` : `Scanned ${scanned} file(s), removed ${removed} missing book(s)`; + if (!failures.length) return base; + const examples = failures + .slice(0, 3) + .map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`) + .join("; "); + const extra = failures.length > 3 ? `; ${failures.length - 3} more` : ""; + return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`; +} + +function errorMessage(error: unknown): string { + if (error instanceof Error && error.message) return truncate(error.message); + return truncate(String(error)); +} + +function truncate(value: string): string { + return value.length > 120 ? `${value.slice(0, 117)}...` : value; +} + function* walkBooks(root: string): Generator { for (const entry of readdirSync(root, { withFileTypes: true })) { const path = join(root, entry.name);