import { Injectable, NotFoundException } from "@nestjs/common"; import { existsSync, readdirSync, statSync } from "node:fs"; import { basename, extname, join } from "node:path"; import { eq, inArray } from "drizzle-orm"; import { DatabaseService } from "../database/database.service.js"; import { automationSettings, books, libraries, series } from "../database/schema.js"; import { JobsService } from "../jobs/jobs.service.js"; import { MetadataService } from "../metadata/metadata.service.js"; import { extractSeriesVolume } from "../metadata/use-cases/extract-series-volume.js"; import { normalizePublishedDate } from "../metadata/use-cases/normalize-published-date.js"; import { extractMetadata } from "./metadata.js"; @Injectable() export class ScannerService { constructor( private readonly database: DatabaseService, private readonly jobs: JobsService, private readonly metadata: MetadataService ) {} enqueueLibraryScan(libraryId: number) { const library = this.database.db.select().from(libraries).where(eq(libraries.id, libraryId)).get(); if (!library) { throw new NotFoundException("Library not found"); } const job = this.jobs.create("library-scan", `Scanning ${library.path}`); setImmediate(() => { void this.scanLibrary(job.id, library).catch((error) => this.jobs.markFailed(job.id, error)); }); return job; } enqueueAllLibrariesScan(detail = "Scanning all enabled libraries") { const job = this.jobs.create("library-scan-all", detail); setImmediate(() => { void this.scanAllLibraries(job.id).catch((error) => this.jobs.markFailed(job.id, error)); }); return job; } enqueueMetadataEnrichment(detail = "Enriching existing books") { const job = this.jobs.create("metadata-enrich", detail); setImmediate(() => { void this.enrichExistingBooks(job.id).catch((error) => this.jobs.markFailed(job.id, error)); }); return job; } private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise { this.jobs.markRunning(jobId, `Scanning ${library.path}`); let count = 0; const failures: ScanFailure[] = []; const seen = new Set(); for (const filePath of walkBooks(library.path)) { seen.add(filePath); try { await this.ingestFile(library.id, filePath); count += 1; } catch (error) { failures.push({ filePath, error: errorMessage(error) }); this.ingestIncompleteFile(library.id, filePath); } } const removed = this.removeMissingBooks(library.id, seen); this.jobs.markSucceeded(jobId, scanDigest(count, removed, failures)); } private async scanAllLibraries(jobId: number): Promise { this.jobs.markRunning(jobId, "Scanning all enabled libraries"); const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all(); let scanned = 0; const failures: ScanFailure[] = []; for (const library of enabledLibraries) { for (const filePath of walkBooks(library.path)) { try { await this.ingestFile(library.id, filePath); scanned += 1; } catch (error) { failures.push({ filePath, error: errorMessage(error) }); this.ingestIncompleteFile(library.id, filePath); } } } this.jobs.markSucceeded(jobId, scanDigest(scanned, 0, failures, `across ${enabledLibraries.length} library/libraries`)); } private async enrichExistingBooks(jobId: number): Promise { this.jobs.markRunning(jobId, "Enriching existing books"); const rows = this.database.db.select({ id: books.id }).from(books).all(); let count = 0; const failures: ScanFailure[] = []; for (const row of rows) { this.markBookEnrichmentStatus(row.id, "running"); try { await this.metadata.enrichBook(row.id); this.markBookEnrichmentStatus(row.id, "succeeded"); count += 1; } catch (error) { this.markBookEnrichmentStatus(row.id, "failed"); failures.push({ filePath: `book #${row.id}`, error: errorMessage(error) }); } } this.jobs.markSucceeded(jobId, enrichmentDigest(count, failures)); } private async ingestFile(libraryId: number, filePath: string): Promise { const stats = statSync(filePath); const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get(); if (existing) { this.database.db .update(books) .set({ scanStatus: "running", enrichmentStatus: "running", updatedAt: this.database.now() }) .where(eq(books.id, existing.id)) .run(); } const localMetadata = await extractMetadata(filePath, this.database.config.storageDir); const now = this.database.now(); const format = bookFormatFromPath(filePath); const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true; const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich }); const seriesInfo = this.resolveSeries(metadata.title, filePath); const values = { libraryId, seriesId: seriesInfo.seriesId, title: metadata.title, author: metadata.author, description: metadata.description, isbn: metadata.isbn, isbn13: metadata.isbn13, identifiersJson: metadata.identifiersJson, localMetadataJson: metadata.localMetadataJson, language: metadata.language, publisher: metadata.publisher, publishedDate: metadata.publishedDate, volumeNumber: seriesInfo.volumeNumber, volumeLabel: seriesInfo.volumeLabel, format, filePath, coverPath: metadata.coverPath, metadataStatus: metadata.metadataStatus, metadataProvenanceJson: metadata.metadataProvenanceJson, scanStatus: "succeeded" as const, enrichmentStatus: shouldRemoteEnrich ? ("succeeded" as const) : ("idle" as const), fileSize: stats.size, fileMtime: stats.mtime.toISOString(), updatedAt: now }; this.upsertBookByFilePath(values, existing, now); } private ingestIncompleteFile(libraryId: number, filePath: string): void { const stats = statSync(filePath); const now = this.database.now(); const existing = this.database.db.select().from(books).where(eq(books.filePath, filePath)).get(); const seriesInfo = this.resolveSeries(basename(filePath, extname(filePath)), filePath); const values = { libraryId, seriesId: seriesInfo.seriesId, title: basename(filePath, extname(filePath)), author: null, description: null, isbn: null, isbn13: null, identifiersJson: JSON.stringify({ candidates: [], isbn10: null, isbn13: null }), localMetadataJson: JSON.stringify({ title: basename(filePath, extname(filePath)), author: null, year: null, isbn: null, fileTitle: basename(filePath, extname(filePath)), raw: { title: basename(filePath, extname(filePath)), author: null, publishedDate: null, fileName: basename(filePath, extname(filePath)) } }), language: null, publisher: null, publishedDate: null, volumeNumber: seriesInfo.volumeNumber, volumeLabel: seriesInfo.volumeLabel, format: bookFormatFromPath(filePath), filePath, coverPath: null, metadataStatus: "none" as const, metadataProvenanceJson: JSON.stringify({ title: "local" }), scanStatus: "failed" as const, enrichmentStatus: "failed" as const, fileSize: stats.size, fileMtime: stats.mtime.toISOString(), updatedAt: now }; this.upsertBookByFilePath(values, existing, now); } private upsertBookByFilePath( values: Omit, existing: typeof books.$inferSelect | undefined, createdAt: string ): void { if (existing) { this.database.db.update(books).set(preserveExistingBookValues(values, existing)).where(eq(books.id, existing.id)).run(); return; } try { this.database.db.insert(books).values({ ...values, createdAt }).run(); return; } catch (error) { if (!isUniqueFilePathError(error)) throw error; const current = this.database.db.select().from(books).where(eq(books.filePath, values.filePath)).get(); if (!current) throw error; this.database.db.update(books).set(preserveExistingBookValues(values, current)).where(eq(books.id, current.id)).run(); } } private removeMissingBooks(libraryId: number, seen: Set): number { const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all(); const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath)); if (!missing.length) return 0; this.database.db.delete(books).where(inArray(books.id, missing.map((book) => book.id))).run(); return missing.length; } private shouldAutoEnrichNewBooks(): boolean { return Boolean( this.database.db .select({ autoEnrichNewBooks: automationSettings.autoEnrichNewBooks }) .from(automationSettings) .where(eq(automationSettings.id, 1)) .get()?.autoEnrichNewBooks ); } private markBookEnrichmentStatus(id: number, enrichmentStatus: "running" | "succeeded" | "failed"): void { this.database.db.update(books).set({ enrichmentStatus, updatedAt: this.database.now() }).where(eq(books.id, id)).run(); } private resolveSeries(title: string, filePath: string): { seriesId: number; volumeNumber: number | null; volumeLabel: string | null } { const parsed = extractSeriesVolume(title, filePath); const now = this.database.now(); const row = this.database.db .insert(series) .values({ title: parsed.seriesTitle, normalizedTitle: parsed.normalizedSeriesTitle, description: null, publisher: null, createdAt: now, updatedAt: now }) .onConflictDoUpdate({ target: series.normalizedTitle, set: { title: parsed.seriesTitle, updatedAt: now } }) .returning({ id: series.id }) .get(); return { seriesId: row.id, volumeNumber: parsed.volumeNumber, volumeLabel: parsed.volumeLabel }; } } type ScanFailure = { filePath: string; error: string; }; export function scanDigest(scanned: number, removed: number, failures: ScanFailure[], suffix?: string): string { const base = suffix ? `Scanned ${scanned} file(s) ${suffix}` : `Scanned ${scanned} file(s), removed ${removed} missing book(s)`; if (!failures.length) return base; const examples = failures .slice(0, 3) .map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`) .join("; "); const extra = failures.length > 3 ? `; ${failures.length - 3} more` : ""; return `${base}, ${failures.length} incomplete file(s): ${examples}${extra}`; } export function enrichmentDigest(enriched: number, failures: ScanFailure[]): string { const base = `Enriched ${enriched} book(s)`; if (!failures.length) return base; const examples = failures .slice(0, 3) .map((failure) => `${basename(failure.filePath)}: ${truncate(failure.error)}`) .join("; "); const extra = failures.length > 3 ? `; ${failures.length - 3} more` : ""; return `${base}, ${failures.length} incomplete book(s): ${examples}${extra}`; } function errorMessage(error: unknown): string { if (error instanceof Error && error.message) return truncate(error.message); return truncate(String(error)); } function truncate(value: string): string { return value.length > 120 ? `${value.slice(0, 117)}...` : value; } function* walkBooks(root: string): Generator { for (const entry of readdirSync(root, { withFileTypes: true })) { const path = join(root, entry.name); if (entry.isDirectory()) { yield* walkBooks(path); continue; } if (!entry.isFile()) continue; const extension = extname(entry.name).toLowerCase(); if (extension === ".epub" || extension === ".pdf" || extension === ".cbz" || extension === ".cbr") { yield path; } } } function bookFormatFromPath(filePath: string): "epub" | "pdf" | "cbz" | "cbr" { const extension = extname(filePath).toLowerCase(); if (extension === ".epub") return "epub"; if (extension === ".cbz") return "cbz"; if (extension === ".cbr") return "cbr"; return "pdf"; } export function preserveExistingBookValues>(values: T, existing: typeof books.$inferSelect): T { const next = { ...values }; if (next.scanStatus === "failed" && existing.title) { next.title = existing.title as never; } for (const field of ["author", "description", "isbn", "isbn13", "language", "publisher", "publishedDate", "coverPath", "seriesId", "volumeNumber", "volumeLabel"] as const) { if (field === "publishedDate") { next.publishedDate = (normalizePublishedDate(next.publishedDate) ?? normalizePublishedDate(existing.publishedDate)) as never; continue; } if (next[field] == null && existing[field] != null) { next[field] = existing[field] as never; } } next.metadataStatus = computeMetadataStatus(next, existing.metadataStatus) as never; next.metadataProvenanceJson = mergeProvenanceJson(String(next.metadataProvenanceJson ?? "{}"), existing.metadataProvenanceJson) as never; return next; } function computeMetadataStatus(values: Partial, existingStatus: string): "enriched" | "partial" | "none" { const hasCover = Boolean(values.coverPath); const filled = [values.author, values.description, values.isbn, values.language, values.publisher, values.publishedDate].filter(Boolean).length; const computed = hasCover && filled >= 2 ? "enriched" : hasCover || filled > 0 ? "partial" : "none"; const rank = { none: 0, partial: 1, enriched: 2 } as const; const safeExisting = existingStatus === "enriched" || existingStatus === "partial" || existingStatus === "none" ? existingStatus : "none"; return rank[computed] >= rank[safeExisting] ? computed : safeExisting; } function mergeProvenanceJson(nextJson: string, existingJson: string | null): string { return JSON.stringify({ ...parseJsonObject(existingJson), ...parseJsonObject(nextJson) }); } function parseJsonObject(value: string | null): Record { if (!value) return {}; try { const parsed = JSON.parse(value) as unknown; return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as Record) : {}; } catch { return {}; } } function isUniqueFilePathError(error: unknown): boolean { return error instanceof Error && error.message.includes("UNIQUE constraint failed: books.file_path"); }