From 48e9459cf3eb6e77bae316dd09dfb761bf0b84c5 Mon Sep 17 00:00:00 2001 From: Git Agent Date: Sun, 23 Aug 2026 13:10:45 +0200 Subject: [PATCH] =?UTF-8?q?feat(api,shared):=20m=C3=A9tadonn=C3=A9es=20mul?= =?UTF-8?q?ti-providers=20et=20automatisation=20des=20scans/enrichissement?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chaîne de résolution metadata (local, Open Library, Google Books, BNF) avec activation/priorité/clé API par provider, colonnes isbn13 et identifiers sur les livres, et intégration au scanner pour compléter métadonnées et jaquettes manquantes. Module automation: réglages persistés, planifications scan/enrichissement et déclenchement manuel, exposés via des endpoints admin. Co-Authored-By: Claude Opus 4.8 --- apps/api/src/admin/admin.controller.ts | 42 +++- apps/api/src/admin/admin.module.ts | 4 +- apps/api/src/app.module.ts | 3 +- apps/api/src/automation/automation.module.ts | 11 + .../src/automation/automation.service.test.ts | 14 ++ apps/api/src/automation/automation.service.ts | 209 ++++++++++++++++++ apps/api/src/books/books.service.ts | 1 + apps/api/src/database/database.service.ts | 75 ++++++- apps/api/src/database/schema.ts | 22 ++ .../api/src/metadata/adapters/bnf.provider.ts | 38 ++++ .../adapters/google-books.provider.ts | 47 ++++ .../src/metadata/adapters/local.provider.ts | 16 ++ .../adapters/open-library.provider.ts | 33 +++ .../src/metadata/extract-identifiers.test.ts | 31 +++ apps/api/src/metadata/metadata.module.ts | 14 ++ apps/api/src/metadata/metadata.service.ts | 167 ++++++++++++++ apps/api/src/metadata/metadata.types.ts | 32 +++ .../metadata/resolve-provider-chain.test.ts | 20 ++ .../metadata/use-cases/extract-identifiers.ts | 114 ++++++++++ .../use-cases/resolve-provider-chain.ts | 14 ++ apps/api/src/scanner/scanner.module.ts | 6 +- apps/api/src/scanner/scanner.service.ts | 87 ++++++-- packages/shared/src/index.ts | 53 +++++ 23 files changed, 1029 insertions(+), 24 deletions(-) create mode 100644 apps/api/src/automation/automation.module.ts create mode 100644 apps/api/src/automation/automation.service.test.ts create mode 100644 apps/api/src/automation/automation.service.ts create mode 100644 apps/api/src/metadata/adapters/bnf.provider.ts create mode 100644 apps/api/src/metadata/adapters/google-books.provider.ts create mode 100644 apps/api/src/metadata/adapters/local.provider.ts create mode 100644 apps/api/src/metadata/adapters/open-library.provider.ts create mode 100644 apps/api/src/metadata/extract-identifiers.test.ts create mode 100644 apps/api/src/metadata/metadata.module.ts create mode 100644 apps/api/src/metadata/metadata.service.ts create mode 100644 apps/api/src/metadata/metadata.types.ts create mode 100644 apps/api/src/metadata/resolve-provider-chain.test.ts create mode 100644 apps/api/src/metadata/use-cases/extract-identifiers.ts create mode 100644 apps/api/src/metadata/use-cases/resolve-provider-chain.ts diff --git a/apps/api/src/admin/admin.controller.ts b/apps/api/src/admin/admin.controller.ts index 401d0d7..9ccad44 100644 --- a/apps/api/src/admin/admin.controller.ts +++ b/apps/api/src/admin/admin.controller.ts @@ -1,9 +1,13 @@ -import { Body, Controller, Delete, Get, Param, Patch, Post, UseGuards } from "@nestjs/common"; +import { Body, Controller, Delete, Get, Param, Patch, Post, Put, UseGuards } from "@nestjs/common"; import { CreateLibraryDto, CreateLibrarySchema, CreateUserDto, CreateUserSchema, + UpdateAutomationSettingsDto, + UpdateAutomationSettingsSchema, + UpdateMetadataSourcesConfigDto, + UpdateMetadataSourcesConfigSchema, UpdateLibraryDto, UpdateLibrarySchema, UpdateUserDto, @@ -13,9 +17,11 @@ import { AuthGuard } from "../auth/auth.guard.js"; import { Roles } from "../auth/roles.decorator.js"; import { RolesGuard } from "../auth/roles.guard.js"; import { AuthService } from "../auth/auth.service.js"; +import { AutomationService } from "../automation/automation.service.js"; import { ZodValidationPipe } from "../common/zod-validation.pipe.js"; import { JobsService } from "../jobs/jobs.service.js"; import { LibrariesService } from "../libraries/libraries.service.js"; +import { MetadataService } from "../metadata/metadata.service.js"; import { ScannerService } from "../scanner/scanner.service.js"; @Controller("admin") @@ -26,7 +32,9 @@ export class AdminController { private readonly auth: AuthService, private readonly libraries: LibrariesService, private readonly jobs: JobsService, - private readonly scanner: ScannerService + private readonly scanner: ScannerService, + private readonly metadata: MetadataService, + private readonly automation: AutomationService ) {} @Get("users") @@ -80,4 +88,34 @@ export class AdminController { listJobs() { return this.jobs.list(); } + + @Get("metadata-sources") + metadataSources() { + return this.metadata.getSourcesConfig(); + } + + @Put("metadata-sources") + updateMetadataSources(@Body(new ZodValidationPipe(UpdateMetadataSourcesConfigSchema)) body: UpdateMetadataSourcesConfigDto) { + return this.metadata.updateSourcesConfig(body); + } + + @Get("automation") + automationSettings() { + return this.automation.getSettings(); + } + + @Put("automation") + updateAutomationSettings(@Body(new ZodValidationPipe(UpdateAutomationSettingsSchema)) body: UpdateAutomationSettingsDto) { + return this.automation.updateSettings(body); + } + + @Post("automation/run-scan") + runAutomationScan() { + return this.automation.runScanNow(); + } + + @Post("automation/run-enrich") + runAutomationEnrich() { + return this.automation.runEnrichNow(); + } } diff --git a/apps/api/src/admin/admin.module.ts b/apps/api/src/admin/admin.module.ts index 71e6d2e..995f4fa 100644 --- a/apps/api/src/admin/admin.module.ts +++ b/apps/api/src/admin/admin.module.ts @@ -1,13 +1,15 @@ import { Module } from "@nestjs/common"; import { AuthModule } from "../auth/auth.module.js"; +import { AutomationModule } from "../automation/automation.module.js"; import { DatabaseModule } from "../database/database.module.js"; import { JobsModule } from "../jobs/jobs.module.js"; import { LibrariesService } from "../libraries/libraries.service.js"; +import { MetadataModule } from "../metadata/metadata.module.js"; import { ScannerModule } from "../scanner/scanner.module.js"; import { AdminController } from "./admin.controller.js"; @Module({ - imports: [AuthModule, DatabaseModule, JobsModule, ScannerModule], + imports: [AuthModule, DatabaseModule, JobsModule, ScannerModule, MetadataModule, AutomationModule], controllers: [AdminController], providers: [LibrariesService] }) diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index df14680..da1de95 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -1,5 +1,6 @@ import { Module } from "@nestjs/common"; import { AdminModule } from "./admin/admin.module.js"; +import { AutomationModule } from "./automation/automation.module.js"; import { AuthModule } from "./auth/auth.module.js"; import { BooksModule } from "./books/books.module.js"; import { DatabaseModule } from "./database/database.module.js"; @@ -8,7 +9,7 @@ import { ScannerModule } from "./scanner/scanner.module.js"; import { HealthController } from "./health.controller.js"; @Module({ - imports: [DatabaseModule, AuthModule, AdminModule, BooksModule, ProgressModule, ScannerModule], + imports: [DatabaseModule, AuthModule, AdminModule, BooksModule, ProgressModule, ScannerModule, AutomationModule], controllers: [HealthController] }) export class AppModule {} diff --git a/apps/api/src/automation/automation.module.ts b/apps/api/src/automation/automation.module.ts new file mode 100644 index 0000000..0f48c8f --- /dev/null +++ b/apps/api/src/automation/automation.module.ts @@ -0,0 +1,11 @@ +import { Module } from "@nestjs/common"; +import { DatabaseModule } from "../database/database.module.js"; +import { ScannerModule } from "../scanner/scanner.module.js"; +import { AutomationService } from "./automation.service.js"; + +@Module({ + imports: [DatabaseModule, ScannerModule], + providers: [AutomationService], + exports: [AutomationService] +}) +export class AutomationModule {} diff --git a/apps/api/src/automation/automation.service.test.ts b/apps/api/src/automation/automation.service.test.ts new file mode 100644 index 0000000..f701068 --- /dev/null +++ b/apps/api/src/automation/automation.service.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { nextRunAt } from "./automation.service.js"; + +describe("automation scheduling", () => { + it("computes the next daily run", () => { + const next = nextRunAt({ frequency: "daily", time: "03:30", dayOfWeek: 1 }, new Date("2026-08-23T02:00:00.000Z")); + expect(next.toISOString()).toBe("2026-08-23T03:30:00.000Z"); + }); + + it("moves elapsed weekly runs to the next week", () => { + const next = nextRunAt({ frequency: "weekly", time: "03:30", dayOfWeek: 0 }, new Date("2026-08-23T04:00:00.000Z")); + expect(next.toISOString()).toBe("2026-08-30T03:30:00.000Z"); + }); +}); diff --git a/apps/api/src/automation/automation.service.ts b/apps/api/src/automation/automation.service.ts new file mode 100644 index 0000000..76c5b87 --- /dev/null +++ b/apps/api/src/automation/automation.service.ts @@ -0,0 +1,209 @@ +import { Injectable, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { Dirent, FSWatcher, readdirSync, watch } from "node:fs"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { AutomationScheduleDto, AutomationSettingsDto, UpdateAutomationSettingsDto } from "@readabook/shared"; +import { DatabaseService } from "../database/database.service.js"; +import { automationSettings, libraries } from "../database/schema.js"; +import { ScannerService } from "../scanner/scanner.service.js"; + +type WatchEntry = { + watchers: FSWatcher[]; + timer: NodeJS.Timeout | null; +}; + +const defaultSchedule: AutomationScheduleDto = { frequency: "disabled", time: "03:00", dayOfWeek: 1 }; + +@Injectable() +export class AutomationService implements OnModuleInit, OnModuleDestroy { + private readonly watchers = new Map(); + private scanTimer: NodeJS.Timeout | null = null; + private enrichTimer: NodeJS.Timeout | null = null; + + constructor( + private readonly database: DatabaseService, + private readonly scanner: ScannerService + ) {} + + onModuleInit(): void { + this.applyRuntimeSettings(); + } + + onModuleDestroy(): void { + this.stopWatchers(); + this.clearSchedules(); + } + + getSettings(): AutomationSettingsDto { + const row = this.readRow(); + return { + watchLibraries: row.watchLibraries, + autoEnrichNewBooks: row.autoEnrichNewBooks, + scanSchedule: parseSchedule(row.scanScheduleJson), + enrichSchedule: parseSchedule(row.enrichScheduleJson) + }; + } + + updateSettings(input: UpdateAutomationSettingsDto): AutomationSettingsDto { + const current = this.getSettings(); + const next: AutomationSettingsDto = { + watchLibraries: input.watchLibraries ?? current.watchLibraries, + autoEnrichNewBooks: input.autoEnrichNewBooks ?? current.autoEnrichNewBooks, + scanSchedule: input.scanSchedule ? normalizeSchedule(input.scanSchedule) : current.scanSchedule, + enrichSchedule: input.enrichSchedule ? normalizeSchedule(input.enrichSchedule) : current.enrichSchedule + }; + this.database.db + .update(automationSettings) + .set({ + watchLibraries: next.watchLibraries, + autoEnrichNewBooks: next.autoEnrichNewBooks, + scanScheduleJson: JSON.stringify(next.scanSchedule), + enrichScheduleJson: JSON.stringify(next.enrichSchedule), + updatedAt: this.database.now() + }) + .where(eq(automationSettings.id, 1)) + .run(); + this.applyRuntimeSettings(); + return this.getSettings(); + } + + runScanNow() { + return this.scanner.enqueueAllLibrariesScan("Manual automation scan"); + } + + runEnrichNow() { + return this.scanner.enqueueMetadataEnrichment("Manual metadata enrichment"); + } + + private applyRuntimeSettings(): void { + const settings = this.getSettings(); + settings.watchLibraries ? this.startWatchers() : this.stopWatchers(); + this.configureSchedules(settings); + } + + private startWatchers(): void { + const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all(); + const enabledIds = new Set(enabledLibraries.map((library) => library.id)); + for (const [id, entry] of this.watchers) { + if (!enabledIds.has(id)) { + entry.watchers.forEach((watcher) => watcher.close()); + if (entry.timer) clearTimeout(entry.timer); + this.watchers.delete(id); + } + } + + for (const library of enabledLibraries) { + if (this.watchers.has(library.id)) continue; + const watchers = watchLibraryDirs(library.path, (_event, filename) => { + if (filename && !isBookPath(String(filename))) return; + const current = this.watchers.get(library.id); + if (!current) return; + if (current.timer) clearTimeout(current.timer); + current.timer = setTimeout(() => { + current.timer = null; + this.scanner.enqueueLibraryScan(library.id); + }, 1500); + }); + for (const watcher of watchers) { + watcher.on("error", () => { + this.watchers.delete(library.id); + }); + } + this.watchers.set(library.id, { watchers, timer: null }); + } + } + + private stopWatchers(): void { + for (const entry of this.watchers.values()) { + entry.watchers.forEach((watcher) => watcher.close()); + if (entry.timer) clearTimeout(entry.timer); + } + this.watchers.clear(); + } + + private configureSchedules(settings: AutomationSettingsDto): void { + this.clearSchedules(); + this.scanTimer = scheduleNext(settings.scanSchedule, () => { + this.scanner.enqueueAllLibrariesScan("Scheduled library scan"); + this.configureSchedules(this.getSettings()); + }); + this.enrichTimer = scheduleNext(settings.enrichSchedule, () => { + this.scanner.enqueueMetadataEnrichment("Scheduled metadata enrichment"); + this.configureSchedules(this.getSettings()); + }); + } + + private clearSchedules(): void { + if (this.scanTimer) clearTimeout(this.scanTimer); + if (this.enrichTimer) clearTimeout(this.enrichTimer); + this.scanTimer = null; + this.enrichTimer = null; + } + + private readRow(): typeof automationSettings.$inferSelect { + return this.database.db.select().from(automationSettings).where(eq(automationSettings.id, 1)).get()!; + } +} + +export function scheduleNext(schedule: AutomationScheduleDto, run: () => void, now = new Date()): NodeJS.Timeout | null { + if (schedule.frequency === "disabled") return null; + const next = nextRunAt(schedule, now); + return setTimeout(run, Math.max(1000, next.getTime() - now.getTime())); +} + +export function nextRunAt(schedule: AutomationScheduleDto, now = new Date()): Date { + const [hour, minute] = schedule.time.split(":").map(Number); + const next = new Date(now); + next.setUTCHours(hour, minute, 0, 0); + if (schedule.frequency === "weekly") { + const delta = (schedule.dayOfWeek - next.getUTCDay() + 7) % 7; + next.setUTCDate(next.getUTCDate() + delta); + } + if (next <= now) { + next.setUTCDate(next.getUTCDate() + (schedule.frequency === "weekly" ? 7 : 1)); + } + return next; +} + +function parseSchedule(value: string): AutomationScheduleDto { + try { + return normalizeSchedule(JSON.parse(value) as Partial); + } catch { + return defaultSchedule; + } +} + +function normalizeSchedule(value: Partial): AutomationScheduleDto { + return { + frequency: value.frequency ?? "disabled", + time: value.time ?? "03:00", + dayOfWeek: value.dayOfWeek ?? 1 + }; +} + +function isBookPath(filePath: string): boolean { + return /\.(epub|pdf|cbz|cbr)$/i.test(filePath); +} + +function watchLibraryDirs(root: string, listener: (event: string, filename: string | Buffer | null) => void): FSWatcher[] { + const watchers: FSWatcher[] = []; + for (const dir of walkDirs(root)) { + watchers.push(watch(dir, listener)); + } + return watchers; +} + +function* walkDirs(root: string): Generator { + yield root; + let entries: Dirent[]; + try { + entries = readdirSync(root, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.isDirectory()) { + yield* walkDirs(join(root, entry.name)); + } + } +} diff --git a/apps/api/src/books/books.service.ts b/apps/api/src/books/books.service.ts index 26d2ad7..0876df1 100644 --- a/apps/api/src/books/books.service.ts +++ b/apps/api/src/books/books.service.ts @@ -125,6 +125,7 @@ function mapBookRow(row: Record) { author: nullable(row.author), description: nullable(row.description), isbn: nullable(row.isbn), + isbn13: nullable(row.isbn13), language: nullable(row.language), publisher: nullable(row.publisher), publishedDate: nullable(row.published_date), diff --git a/apps/api/src/database/database.service.ts b/apps/api/src/database/database.service.ts index 9c26a7b..95bd4bd 100644 --- a/apps/api/src/database/database.service.ts +++ b/apps/api/src/database/database.service.ts @@ -56,6 +56,8 @@ export class DatabaseService implements OnModuleDestroy { author TEXT, description TEXT, isbn TEXT, + isbn13 TEXT, + identifiers_json TEXT, language TEXT, publisher TEXT, published_date TEXT, @@ -89,6 +91,26 @@ export class DatabaseService implements OnModuleDestroy { updated_at TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS metadata_source_config ( + provider TEXT PRIMARY KEY CHECK (provider IN ('local','openlibrary','googlebooks','bnf')), + enabled INTEGER NOT NULL DEFAULT 1, + priority INTEGER NOT NULL, + api_key TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS automation_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), + watch_libraries INTEGER NOT NULL DEFAULT 0, + auto_enrich_new_books INTEGER NOT NULL DEFAULT 1, + isbn_priority_enabled INTEGER NOT NULL DEFAULT 1, + scan_schedule_json TEXT NOT NULL, + enrich_schedule_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE VIRTUAL TABLE IF NOT EXISTS book_fts USING fts5( title, author, @@ -100,6 +122,7 @@ export class DatabaseService implements OnModuleDestroy { CREATE INDEX IF NOT EXISTS books_library_idx ON books(library_id); CREATE INDEX IF NOT EXISTS books_title_idx ON books(title); + CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13); CREATE INDEX IF NOT EXISTS jobs_status_idx ON jobs(status); CREATE TRIGGER IF NOT EXISTS books_ai AFTER INSERT ON books BEGIN @@ -120,6 +143,8 @@ export class DatabaseService implements OnModuleDestroy { END; `); this.ensureBooksSupportsComicArchives(); + this.ensureBooksMetadataColumns(); + this.ensureMetadataDefaults(); this.sqlite.exec("INSERT INTO book_fts(book_fts) VALUES('rebuild')"); } @@ -146,6 +171,8 @@ export class DatabaseService implements OnModuleDestroy { author TEXT, description TEXT, isbn TEXT, + isbn13 TEXT, + identifiers_json TEXT, language TEXT, publisher TEXT, published_date TEXT, @@ -158,11 +185,11 @@ export class DatabaseService implements OnModuleDestroy { updated_at TEXT NOT NULL ); INSERT INTO books ( - id, library_id, title, author, description, isbn, language, publisher, published_date, + id, library_id, title, author, description, isbn, isbn13, identifiers_json, language, publisher, published_date, format, file_path, cover_path, file_size, file_mtime, created_at, updated_at ) SELECT - id, library_id, title, author, description, isbn, language, publisher, published_date, + id, library_id, title, author, description, isbn, NULL, NULL, language, publisher, published_date, format, file_path, cover_path, file_size, file_mtime, created_at, updated_at FROM books_legacy_format; DROP TABLE books_legacy_format; @@ -174,6 +201,7 @@ export class DatabaseService implements OnModuleDestroy { CREATE UNIQUE INDEX IF NOT EXISTS books_file_path_unique ON books(file_path); CREATE INDEX IF NOT EXISTS books_library_idx ON books(library_id); CREATE INDEX IF NOT EXISTS books_title_idx ON books(title); + CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13); CREATE TRIGGER IF NOT EXISTS books_ai AFTER INSERT ON books BEGIN INSERT INTO book_fts(rowid, title, author, description, isbn) @@ -193,4 +221,47 @@ export class DatabaseService implements OnModuleDestroy { END; `); } + + private ensureBooksMetadataColumns(): void { + const columns = this.sqlite.prepare("PRAGMA table_info(books)").all() as Array<{ name: string }>; + const names = new Set(columns.map((column) => column.name)); + if (!names.has("isbn13")) { + this.sqlite.exec("ALTER TABLE books ADD COLUMN isbn13 TEXT"); + } + if (!names.has("identifiers_json")) { + this.sqlite.exec("ALTER TABLE books ADD COLUMN identifiers_json TEXT"); + } + this.sqlite.exec("CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13)"); + } + + private ensureMetadataDefaults(): void { + const now = this.now(); + const insertSource = this.sqlite.prepare(` + INSERT INTO metadata_source_config (provider, enabled, priority, api_key, created_at, updated_at) + VALUES (?, ?, ?, NULL, ?, ?) + ON CONFLICT(provider) DO NOTHING + `); + insertSource.run("local", 1, 0, now, now); + insertSource.run("openlibrary", this.config.openLibraryEnabled ? 1 : 0, 1, now, now); + insertSource.run("googlebooks", 0, 2, now, now); + insertSource.run("bnf", 0, 3, now, now); + + this.sqlite + .prepare( + ` + INSERT INTO automation_settings ( + id, watch_libraries, auto_enrich_new_books, isbn_priority_enabled, + scan_schedule_json, enrich_schedule_json, created_at, updated_at + ) + VALUES (1, 0, 1, 1, ?, ?, ?, ?) + ON CONFLICT(id) DO NOTHING + ` + ) + .run( + JSON.stringify({ frequency: "disabled", time: "03:00", dayOfWeek: 1 }), + JSON.stringify({ frequency: "disabled", time: "04:00", dayOfWeek: 1 }), + now, + now + ); + } } diff --git a/apps/api/src/database/schema.ts b/apps/api/src/database/schema.ts index cd04364..87800de 100644 --- a/apps/api/src/database/schema.ts +++ b/apps/api/src/database/schema.ts @@ -34,6 +34,8 @@ export const books = sqliteTable( author: text("author"), description: text("description"), isbn: text("isbn"), + isbn13: text("isbn13"), + identifiersJson: text("identifiers_json"), language: text("language"), publisher: text("publisher"), publishedDate: text("published_date"), @@ -75,3 +77,23 @@ export const jobs = sqliteTable("jobs", { createdAt: text("created_at").notNull(), updatedAt: text("updated_at").notNull() }); + +export const metadataSourceConfig = sqliteTable("metadata_source_config", { + provider: text("provider", { enum: ["local", "openlibrary", "googlebooks", "bnf"] }).primaryKey(), + enabled: integer("enabled", { mode: "boolean" }).notNull().default(true), + priority: integer("priority").notNull(), + apiKey: text("api_key"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull() +}); + +export const automationSettings = sqliteTable("automation_settings", { + id: integer("id").primaryKey(), + watchLibraries: integer("watch_libraries", { mode: "boolean" }).notNull().default(false), + autoEnrichNewBooks: integer("auto_enrich_new_books", { mode: "boolean" }).notNull().default(true), + isbnPriorityEnabled: integer("isbn_priority_enabled", { mode: "boolean" }).notNull().default(true), + scanScheduleJson: text("scan_schedule_json").notNull(), + enrichScheduleJson: text("enrich_schedule_json").notNull(), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull() +}); diff --git a/apps/api/src/metadata/adapters/bnf.provider.ts b/apps/api/src/metadata/adapters/bnf.provider.ts new file mode 100644 index 0000000..7294f86 --- /dev/null +++ b/apps/api/src/metadata/adapters/bnf.provider.ts @@ -0,0 +1,38 @@ +import { Injectable } from "@nestjs/common"; +import { XMLParser } from "fast-xml-parser"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; + +const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); + +@Injectable() +export class BnfProvider implements MetadataProvider { + readonly id = "bnf" as const; + + async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise { + const query = lookup.identifiers.isbn13 + ? `bib.isbn all "${lookup.identifiers.isbn13}"` + : `bib.title all "${lookup.title.replace(/"/g, " ")}"`; + const url = new URL("https://catalogue.bnf.fr/api/SRU"); + url.searchParams.set("version", "1.2"); + url.searchParams.set("operation", "searchRetrieve"); + url.searchParams.set("query", query); + url.searchParams.set("maximumRecords", "1"); + + const response = await fetch(url, { signal: AbortSignal.timeout(5000) }); + if (!response.ok) return null; + const parsed = parser.parse(await response.text()); + const record = parsed?.searchRetrieveResponse?.records?.record?.recordData; + if (!record) return null; + const text = JSON.stringify(record); + return { + title: match(text, /"titleInfo"[^}]*"title":"([^"]+)"/), + author: match(text, /"namePart":"([^"]+)"/), + publisher: match(text, /"publisher":"([^"]+)"/), + publishedDate: match(text, /"dateIssued":"([^"]+)"/) + }; + } +} + +function match(value: string, pattern: RegExp): string | undefined { + return value.match(pattern)?.[1]; +} diff --git a/apps/api/src/metadata/adapters/google-books.provider.ts b/apps/api/src/metadata/adapters/google-books.provider.ts new file mode 100644 index 0000000..228241d --- /dev/null +++ b/apps/api/src/metadata/adapters/google-books.provider.ts @@ -0,0 +1,47 @@ +import { Injectable } from "@nestjs/common"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; + +@Injectable() +export class GoogleBooksProvider implements MetadataProvider { + readonly id = "googlebooks" as const; + + async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise { + const query = lookup.identifiers.isbn13 + ? `isbn:${lookup.identifiers.isbn13}` + : `intitle:${lookup.title}${lookup.author ? `+inauthor:${lookup.author}` : ""}`; + const url = new URL("https://www.googleapis.com/books/v1/volumes"); + url.searchParams.set("q", query); + url.searchParams.set("maxResults", "1"); + if (config.apiKey) url.searchParams.set("key", config.apiKey); + + const response = await fetch(url, { signal: AbortSignal.timeout(4000) }); + if (!response.ok) return null; + const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record }> }; + const info = data.items?.[0]?.volumeInfo; + if (!info) return null; + return { + title: stringValue(info.title) ?? undefined, + author: arrayJoin(info.authors), + description: stringValue(info.description), + language: stringValue(info.language), + publisher: stringValue(info.publisher), + publishedDate: stringValue(info.publishedDate), + isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers) + }; + } +} + +function stringValue(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function arrayJoin(value: unknown): string | null { + return Array.isArray(value) && value.length ? value.map(String).join(", ") : null; +} + +function isbnFromIndustryIdentifiers(value: unknown): string | null { + if (!Array.isArray(value)) return null; + const isbn13 = value.find((entry) => entry?.type === "ISBN_13")?.identifier; + const isbn10 = value.find((entry) => entry?.type === "ISBN_10")?.identifier; + return stringValue(isbn13) ?? stringValue(isbn10); +} diff --git a/apps/api/src/metadata/adapters/local.provider.ts b/apps/api/src/metadata/adapters/local.provider.ts new file mode 100644 index 0000000..5223716 --- /dev/null +++ b/apps/api/src/metadata/adapters/local.provider.ts @@ -0,0 +1,16 @@ +import { Injectable } from "@nestjs/common"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; + +@Injectable() +export class LocalMetadataProvider implements MetadataProvider { + readonly id = "local" as const; + + async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise { + return { + title: lookup.title, + author: lookup.author, + isbn: lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10, + identifiers: lookup.identifiers + }; + } +} diff --git a/apps/api/src/metadata/adapters/open-library.provider.ts b/apps/api/src/metadata/adapters/open-library.provider.ts new file mode 100644 index 0000000..17ed923 --- /dev/null +++ b/apps/api/src/metadata/adapters/open-library.provider.ts @@ -0,0 +1,33 @@ +import { Injectable } from "@nestjs/common"; +import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; + +@Injectable() +export class OpenLibraryProvider implements MetadataProvider { + readonly id = "openlibrary" as const; + + async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise { + const query = lookup.identifiers.isbn13 + ? `isbn:${encodeURIComponent(lookup.identifiers.isbn13)}` + : `title:${encodeURIComponent(lookup.title)}${lookup.author ? ` author:${encodeURIComponent(lookup.author)}` : ""}`; + const response = await fetch(`https://openlibrary.org/search.json?q=${query}&limit=1`, { + headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" }, + signal: AbortSignal.timeout(4000) + }); + if (!response.ok) return null; + const data = (await response.json()) as { docs?: Array> }; + const doc = data.docs?.[0]; + if (!doc) return null; + return { + author: firstArrayValue(doc.author_name), + language: firstArrayValue(doc.language), + publisher: firstArrayValue(doc.publisher), + publishedDate: String(doc.first_publish_year ?? "") || null, + isbn: firstArrayValue(doc.isbn) + }; + } +} + +function firstArrayValue(value: unknown): string | null { + if (!Array.isArray(value) || !value.length) return null; + return String(value[0]); +} diff --git a/apps/api/src/metadata/extract-identifiers.test.ts b/apps/api/src/metadata/extract-identifiers.test.ts new file mode 100644 index 0000000..56b1ad4 --- /dev/null +++ b/apps/api/src/metadata/extract-identifiers.test.ts @@ -0,0 +1,31 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { ExtractIdentifiers, normalizeIsbn, toIsbn13 } from "./use-cases/extract-identifiers.js"; + +describe("ISBN normalization", () => { + it("accepts valid ISBN-10/13 and rejects bad checksums", () => { + expect(normalizeIsbn("0-306-40615-2")).toBe("0306406152"); + expect(normalizeIsbn("978-0-306-40615-7")).toBe("9780306406157"); + expect(normalizeIsbn("978-0-306-40615-8")).toBeNull(); + }); + + it("converts ISBN-10 to ISBN-13", () => { + expect(toIsbn13("0-306-40615-2")).toBe("9780306406157"); + }); +}); + +describe("ExtractIdentifiers", () => { + it("uses embedded PDF text before filename fallback candidates", () => { + const dir = mkdtempSync(join(tmpdir(), "readabook-isbn-")); + const file = join(dir, "fallback 9780306406157.pdf"); + writeFileSync(file, "%PDF-1.4\n1 0 obj << /Title (Book) /Subject (ISBN 0-306-40615-2) >> endobj"); + + const identifiers = new ExtractIdentifiers().fromMetadataAndFile({ isbn: null }, file); + + expect(identifiers.isbn10).toBe("0306406152"); + expect(identifiers.isbn13).toBe("9780306406157"); + expect(identifiers.candidates).toContain("9780306406157"); + }); +}); diff --git a/apps/api/src/metadata/metadata.module.ts b/apps/api/src/metadata/metadata.module.ts new file mode 100644 index 0000000..d15002a --- /dev/null +++ b/apps/api/src/metadata/metadata.module.ts @@ -0,0 +1,14 @@ +import { Module } from "@nestjs/common"; +import { DatabaseModule } from "../database/database.module.js"; +import { BnfProvider } from "./adapters/bnf.provider.js"; +import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; +import { LocalMetadataProvider } from "./adapters/local.provider.js"; +import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; +import { MetadataService } from "./metadata.service.js"; + +@Module({ + imports: [DatabaseModule], + providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider], + exports: [MetadataService] +}) +export class MetadataModule {} diff --git a/apps/api/src/metadata/metadata.service.ts b/apps/api/src/metadata/metadata.service.ts new file mode 100644 index 0000000..b27001d --- /dev/null +++ b/apps/api/src/metadata/metadata.service.ts @@ -0,0 +1,167 @@ +import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; +import { eq } from "drizzle-orm"; +import { + MetadataSourcesConfigDto, + UpdateMetadataSourcesConfigDto +} from "@readabook/shared"; +import { DatabaseService } from "../database/database.service.js"; +import { automationSettings, books, metadataSourceConfig } from "../database/schema.js"; +import { BookMetadata } from "../scanner/metadata.js"; +import { BnfProvider } from "./adapters/bnf.provider.js"; +import { GoogleBooksProvider } from "./adapters/google-books.provider.js"; +import { LocalMetadataProvider } from "./adapters/local.provider.js"; +import { OpenLibraryProvider } from "./adapters/open-library.provider.js"; +import { MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js"; +import { ExtractIdentifiers, toIsbn13 } from "./use-cases/extract-identifiers.js"; +import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js"; + +@Injectable() +export class MetadataService { + private readonly extractIdentifiers = new ExtractIdentifiers(); + private readonly resolveProviderChain: ResolveProviderChain; + + constructor( + private readonly database: DatabaseService, + local: LocalMetadataProvider, + openLibrary: OpenLibraryProvider, + googleBooks: GoogleBooksProvider, + bnf: BnfProvider + ) { + this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf]); + } + + getSourcesConfig(): MetadataSourcesConfigDto { + const settings = this.getAutomationRow(); + return { + isbnPriorityEnabled: Boolean(settings.isbnPriorityEnabled), + sources: this.getProviderConfigs().map((source) => ({ + provider: source.provider, + enabled: source.provider === "local" ? true : source.enabled, + priority: source.priority, + hasApiKey: Boolean(source.apiKey) + })) + }; + } + + updateSourcesConfig(input: UpdateMetadataSourcesConfigDto): MetadataSourcesConfigDto { + const now = this.database.now(); + if (input.isbnPriorityEnabled !== undefined) { + this.database.db + .update(automationSettings) + .set({ isbnPriorityEnabled: input.isbnPriorityEnabled, updatedAt: now }) + .where(eq(automationSettings.id, 1)) + .run(); + } + for (const source of input.sources ?? []) { + if ((source.provider as string) === "local") { + throw new BadRequestException("Local metadata source is always active and cannot be updated"); + } + const values: Partial = { + enabled: source.enabled, + priority: source.priority, + updatedAt: now + }; + if (source.apiKey !== undefined) values.apiKey = source.apiKey; + this.database.db.update(metadataSourceConfig).set(values).where(eq(metadataSourceConfig.provider, source.provider)).run(); + } + return this.getSourcesConfig(); + } + + async enrichMetadata(localMetadata: BookMetadata, filePath: string, options: { remote: boolean }): Promise { + const identifiers = this.extractIdentifiers.fromMetadataAndFile(localMetadata, filePath); + const configs = this.getProviderConfigs(); + const chain = options.remote + ? this.resolveProviderChain.resolve(configs) + : this.resolveProviderChain.resolve(configs).filter((entry) => entry.provider.id === "local"); + let merged: BookMetadata = { ...localMetadata }; + + for (const { provider, config } of chain) { + try { + const match = await provider.lookup( + { + title: merged.title, + author: merged.author, + filePath, + identifiers + }, + config + ); + if (match) merged = mergeMetadata(merged, match); + } catch { + // Provider failures must not block local ingestion. + } + } + + const isbn13 = identifiers.isbn13 ?? (merged.isbn ? toIsbn13(merged.isbn) : null); + return { + ...merged, + isbn: merged.isbn ?? isbn13 ?? identifiers.isbn10, + isbn13, + identifiersJson: JSON.stringify(identifiers) + }; + } + + async enrichBook(bookId: number): Promise { + const book = this.database.db.select().from(books).where(eq(books.id, bookId)).get(); + if (!book) throw new NotFoundException("Book not found"); + const metadata: BookMetadata = { + title: book.title, + author: book.author, + description: book.description, + isbn: book.isbn, + language: book.language, + publisher: book.publisher, + publishedDate: book.publishedDate, + coverPath: book.coverPath + }; + const enriched = await this.enrichMetadata(metadata, book.filePath, { remote: true }); + return this.database.db + .update(books) + .set({ + title: enriched.title, + author: enriched.author, + description: enriched.description, + isbn: enriched.isbn, + isbn13: enriched.isbn13, + identifiersJson: enriched.identifiersJson, + language: enriched.language, + publisher: enriched.publisher, + publishedDate: enriched.publishedDate, + coverPath: enriched.coverPath, + updatedAt: this.database.now() + }) + .where(eq(books.id, book.id)) + .returning() + .get(); + } + + private getProviderConfigs(): MetadataProviderConfig[] { + return this.database.db + .select() + .from(metadataSourceConfig) + .all() + .map((row) => ({ + provider: row.provider, + enabled: row.provider === "local" ? true : row.enabled, + priority: row.provider === "local" ? 0 : row.priority, + apiKey: row.apiKey + })); + } + + private getAutomationRow(): typeof automationSettings.$inferSelect { + return this.database.db.select().from(automationSettings).where(eq(automationSettings.id, 1)).get()!; + } +} + +function mergeMetadata(current: BookMetadata, next: MetadataMatch): BookMetadata { + return { + title: next.title ?? current.title, + author: current.author ?? next.author ?? null, + description: current.description ?? next.description ?? null, + isbn: current.isbn ?? next.isbn ?? null, + language: current.language ?? next.language ?? null, + publisher: current.publisher ?? next.publisher ?? null, + publishedDate: current.publishedDate ?? next.publishedDate ?? null, + coverPath: current.coverPath ?? next.coverPath ?? null + }; +} diff --git a/apps/api/src/metadata/metadata.types.ts b/apps/api/src/metadata/metadata.types.ts new file mode 100644 index 0000000..0e1521c --- /dev/null +++ b/apps/api/src/metadata/metadata.types.ts @@ -0,0 +1,32 @@ +import { BookMetadata } from "../scanner/metadata.js"; + +export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf"; + +export type BookIdentifiers = { + isbn10: string | null; + isbn13: string | null; + candidates: string[]; +}; + +export type MetadataLookup = { + title: string; + author: string | null; + filePath: string; + identifiers: BookIdentifiers; +}; + +export type MetadataMatch = Partial & { + identifiers?: Partial; +}; + +export type MetadataProviderConfig = { + provider: MetadataProviderId; + enabled: boolean; + priority: number; + apiKey: string | null; +}; + +export interface MetadataProvider { + readonly id: MetadataProviderId; + lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise; +} diff --git a/apps/api/src/metadata/resolve-provider-chain.test.ts b/apps/api/src/metadata/resolve-provider-chain.test.ts new file mode 100644 index 0000000..b26176b --- /dev/null +++ b/apps/api/src/metadata/resolve-provider-chain.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { MetadataProvider } from "./metadata.types.js"; +import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js"; + +const provider = (id: MetadataProvider["id"]): MetadataProvider => ({ + id, + lookup: async () => null +}); + +describe("ResolveProviderChain", () => { + it("keeps local active and orders enabled remote providers by priority", () => { + const chain = new ResolveProviderChain([provider("googlebooks"), provider("local"), provider("bnf")]).resolve([ + { provider: "local", enabled: false, priority: 99, apiKey: null }, + { provider: "googlebooks", enabled: true, priority: 2, apiKey: null }, + { provider: "bnf", enabled: true, priority: 1, apiKey: null } + ]); + + expect(chain.map((entry) => entry.provider.id)).toEqual(["bnf", "googlebooks", "local"]); + }); +}); diff --git a/apps/api/src/metadata/use-cases/extract-identifiers.ts b/apps/api/src/metadata/use-cases/extract-identifiers.ts new file mode 100644 index 0000000..a1cec41 --- /dev/null +++ b/apps/api/src/metadata/use-cases/extract-identifiers.ts @@ -0,0 +1,114 @@ +import { readFileSync } from "node:fs"; +import { basename, extname } from "node:path"; +import AdmZip from "adm-zip"; +import { XMLParser } from "fast-xml-parser"; + +export type ExtractedIdentifiers = { + isbn10: string | null; + isbn13: string | null; + candidates: string[]; +}; + +const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + textNodeName: "#text" +}); + +export class ExtractIdentifiers { + fromMetadataAndFile(metadata: { isbn?: string | null }, filePath: string): ExtractedIdentifiers { + const candidates = new Set(); + for (const value of [metadata.isbn, basename(filePath, extname(filePath))]) { + for (const isbn of findIsbns(String(value ?? ""))) candidates.add(isbn); + } + + const extension = extname(filePath).toLowerCase(); + if (extension === ".epub") { + for (const isbn of findIsbns(readLimitedEpubText(filePath))) candidates.add(isbn); + } + if (extension === ".pdf") { + const buffer = readFileSync(filePath); + const head = buffer.subarray(0, Math.min(buffer.length, 256 * 1024)).toString("latin1"); + for (const isbn of findIsbns(head)) candidates.add(isbn); + } + + return normalizeCandidates([...candidates]); + } +} + +export function normalizeIsbn(value: string): string | null { + const compact = value.replace(/[^0-9X]/gi, "").toUpperCase(); + if (compact.length === 10 && isValidIsbn10(compact)) return compact; + if (compact.length === 13 && isValidIsbn13(compact)) return compact; + return null; +} + +export function toIsbn13(value: string): string | null { + const isbn = normalizeIsbn(value); + if (!isbn) return null; + if (isbn.length === 13) return isbn; + const stem = `978${isbn.slice(0, 9)}`; + let sum = 0; + for (let index = 0; index < stem.length; index += 1) { + sum += Number(stem[index]) * (index % 2 === 0 ? 1 : 3); + } + return `${stem}${(10 - (sum % 10)) % 10}`; +} + +function normalizeCandidates(values: string[]): ExtractedIdentifiers { + const normalized = [...new Set(values.map(normalizeIsbn).filter((value): value is string => Boolean(value)))]; + const isbn13 = normalized.map(toIsbn13).find((value): value is string => Boolean(value)) ?? null; + const isbn10 = normalized.find((value) => value.length === 10) ?? null; + return { isbn10, isbn13, candidates: normalized }; +} + +function findIsbns(text: string): string[] { + const matches = text.match(/(?:ISBN(?:-1[03])?:?\s*)?(?:97[89][-\s]?)?(?:\d[-\s]?){9,12}[\dX]/gi) ?? []; + return matches.map((match) => match.replace(/^ISBN(?:-1[03])?:?\s*/i, "")); +} + +function isValidIsbn10(value: string): boolean { + let sum = 0; + for (let index = 0; index < 10; index += 1) { + const char = value[index]; + const digit = char === "X" && index === 9 ? 10 : Number(char); + if (!Number.isInteger(digit)) return false; + sum += digit * (10 - index); + } + return sum % 11 === 0; +} + +function isValidIsbn13(value: string): boolean { + let sum = 0; + for (let index = 0; index < 13; index += 1) { + const digit = Number(value[index]); + if (!Number.isInteger(digit)) return false; + sum += digit * (index % 2 === 0 ? 1 : 3); + } + return sum % 10 === 0; +} + +function readLimitedEpubText(filePath: string): string { + try { + const zip = new AdmZip(filePath); + const fragments: string[] = [zip.readAsText("META-INF/container.xml")]; + for (const entry of zip.getEntries()) { + if (fragments.join("").length > 256 * 1024) break; + if (!entry.isDirectory && /\.(opf|xhtml|html|htm|xml)$/i.test(entry.entryName)) { + fragments.push(stripXml(zip.readAsText(entry))); + } + } + return fragments.join("\n"); + } catch { + return ""; + } +} + +function stripXml(value: string): string { + try { + const parsed = xmlParser.parse(value); + return JSON.stringify(parsed).slice(0, 256 * 1024); + } catch { + return value.slice(0, 256 * 1024); + } +} diff --git a/apps/api/src/metadata/use-cases/resolve-provider-chain.ts b/apps/api/src/metadata/use-cases/resolve-provider-chain.ts new file mode 100644 index 0000000..0314ae2 --- /dev/null +++ b/apps/api/src/metadata/use-cases/resolve-provider-chain.ts @@ -0,0 +1,14 @@ +import { MetadataProvider, MetadataProviderConfig } from "../metadata.types.js"; + +export class ResolveProviderChain { + constructor(private readonly providers: MetadataProvider[]) {} + + resolve(configs: MetadataProviderConfig[]): Array<{ provider: MetadataProvider; config: MetadataProviderConfig }> { + const configById = new Map(configs.map((config) => [config.provider, config])); + return this.providers + .map((provider) => ({ provider, config: configById.get(provider.id) })) + .filter((entry): entry is { provider: MetadataProvider; config: MetadataProviderConfig } => Boolean(entry.config)) + .filter((entry) => entry.config.provider === "local" || entry.config.enabled) + .sort((left, right) => left.config.priority - right.config.priority); + } +} diff --git a/apps/api/src/scanner/scanner.module.ts b/apps/api/src/scanner/scanner.module.ts index 5c63116..3970304 100644 --- a/apps/api/src/scanner/scanner.module.ts +++ b/apps/api/src/scanner/scanner.module.ts @@ -1,12 +1,12 @@ import { Module } from "@nestjs/common"; import { DatabaseModule } from "../database/database.module.js"; import { JobsModule } from "../jobs/jobs.module.js"; -import { OpenLibraryService } from "./open-library.service.js"; +import { MetadataModule } from "../metadata/metadata.module.js"; import { ScannerService } from "./scanner.service.js"; @Module({ - imports: [DatabaseModule, JobsModule], - providers: [ScannerService, OpenLibraryService], + imports: [DatabaseModule, JobsModule, MetadataModule], + providers: [ScannerService], exports: [ScannerService] }) export class ScannerModule {} diff --git a/apps/api/src/scanner/scanner.service.ts b/apps/api/src/scanner/scanner.service.ts index 90924fe..d6ad412 100644 --- a/apps/api/src/scanner/scanner.service.ts +++ b/apps/api/src/scanner/scanner.service.ts @@ -1,19 +1,19 @@ import { Injectable, NotFoundException } from "@nestjs/common"; -import { readdirSync, statSync } from "node:fs"; +import { existsSync, readdirSync, statSync } from "node:fs"; import { extname, join } from "node:path"; -import { eq } from "drizzle-orm"; +import { eq, inArray } from "drizzle-orm"; import { DatabaseService } from "../database/database.service.js"; -import { books, libraries } from "../database/schema.js"; +import { automationSettings, books, libraries } from "../database/schema.js"; import { JobsService } from "../jobs/jobs.service.js"; +import { MetadataService } from "../metadata/metadata.service.js"; import { extractMetadata } from "./metadata.js"; -import { OpenLibraryService } from "./open-library.service.js"; @Injectable() export class ScannerService { constructor( private readonly database: DatabaseService, private readonly jobs: JobsService, - private readonly openLibrary: OpenLibraryService + private readonly metadata: MetadataService ) {} enqueueLibraryScan(libraryId: number) { @@ -28,36 +28,75 @@ export class ScannerService { return job; } + enqueueAllLibrariesScan(detail = "Scanning all enabled libraries") { + const job = this.jobs.create("library-scan-all", detail); + setImmediate(() => { + void this.scanAllLibraries(job.id).catch((error) => this.jobs.markFailed(job.id, error)); + }); + return job; + } + + enqueueMetadataEnrichment(detail = "Enriching existing books") { + const job = this.jobs.create("metadata-enrich", detail); + setImmediate(() => { + void this.enrichExistingBooks(job.id).catch((error) => this.jobs.markFailed(job.id, error)); + }); + return job; + } + private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise { this.jobs.markRunning(jobId, `Scanning ${library.path}`); let count = 0; + const seen = new Set(); for (const filePath of walkBooks(library.path)) { + seen.add(filePath); await this.ingestFile(library.id, filePath); count += 1; } - this.jobs.markSucceeded(jobId, `Scanned ${count} file(s)`); + const removed = this.removeMissingBooks(library.id, seen); + this.jobs.markSucceeded(jobId, `Scanned ${count} file(s), removed ${removed} missing book(s)`); + } + + private async scanAllLibraries(jobId: number): Promise { + this.jobs.markRunning(jobId, "Scanning all enabled libraries"); + const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all(); + let scanned = 0; + for (const library of enabledLibraries) { + for (const filePath of walkBooks(library.path)) { + await this.ingestFile(library.id, filePath); + scanned += 1; + } + } + this.jobs.markSucceeded(jobId, `Scanned ${scanned} file(s) across ${enabledLibraries.length} library/libraries`); + } + + private async enrichExistingBooks(jobId: number): Promise { + this.jobs.markRunning(jobId, "Enriching existing books"); + const rows = this.database.db.select({ id: books.id }).from(books).all(); + let count = 0; + for (const row of rows) { + await this.metadata.enrichBook(row.id); + count += 1; + } + this.jobs.markSucceeded(jobId, `Enriched ${count} book(s)`); } private async ingestFile(libraryId: number, filePath: string): Promise { const stats = statSync(filePath); - let metadata = await extractMetadata(filePath, this.database.config.storageDir); - if (this.database.config.openLibraryEnabled) { - try { - metadata = { ...metadata, ...(await this.openLibrary.enrich(metadata)) }; - } catch { - // Remote enrichment is opportunistic; local ingestion must stay deterministic. - } - } - + const localMetadata = await extractMetadata(filePath, this.database.config.storageDir); const now = this.database.now(); const format = bookFormatFromPath(filePath); const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get(); + const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true; + const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich }); const values = { libraryId, title: metadata.title, author: metadata.author, description: metadata.description, isbn: metadata.isbn, + isbn13: metadata.isbn13, + identifiersJson: metadata.identifiersJson, language: metadata.language, publisher: metadata.publisher, publishedDate: metadata.publishedDate, @@ -73,6 +112,24 @@ export class ScannerService { ? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get() : this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get(); } + + private removeMissingBooks(libraryId: number, seen: Set): number { + const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all(); + const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath)); + if (!missing.length) return 0; + this.database.db.delete(books).where(inArray(books.id, missing.map((book) => book.id))).run(); + return missing.length; + } + + private shouldAutoEnrichNewBooks(): boolean { + return Boolean( + this.database.db + .select({ autoEnrichNewBooks: automationSettings.autoEnrichNewBooks }) + .from(automationSettings) + .where(eq(automationSettings.id, 1)) + .get()?.autoEnrichNewBooks + ); + } } function* walkBooks(root: string): Generator { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 6acab3f..9243ebd 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -91,6 +91,7 @@ export const BookSchema = z.object({ author: z.string().nullable(), description: z.string().nullable(), isbn: z.string().nullable(), + isbn13: z.string().nullable(), language: z.string().nullable(), publisher: z.string().nullable(), publishedDate: z.string().nullable(), @@ -137,3 +138,55 @@ export const JobSchema = z.object({ updatedAt: z.string() }); export type JobDto = z.infer; + +export const MetadataProviderIdSchema = z.enum(["local", "openlibrary", "googlebooks", "bnf"]); +export type MetadataProviderId = z.infer; + +export const MetadataSourceConfigSchema = z.object({ + provider: MetadataProviderIdSchema, + enabled: z.boolean(), + priority: z.number().int().min(0), + hasApiKey: z.boolean().default(false) +}); +export type MetadataSourceConfigDto = z.infer; + +export const MetadataSourcesConfigSchema = z.object({ + isbnPriorityEnabled: z.boolean(), + sources: z.array(MetadataSourceConfigSchema) +}); +export type MetadataSourcesConfigDto = z.infer; + +export const UpdateMetadataSourceConfigSchema = z.object({ + provider: MetadataProviderIdSchema.exclude(["local"]), + enabled: z.boolean(), + priority: z.number().int().min(1).max(100), + apiKey: z.string().min(1).nullable().optional() +}); +export type UpdateMetadataSourceConfigDto = z.infer; + +export const UpdateMetadataSourcesConfigSchema = z.object({ + isbnPriorityEnabled: z.boolean().optional(), + sources: z.array(UpdateMetadataSourceConfigSchema).optional() +}); +export type UpdateMetadataSourcesConfigDto = z.infer; + +export const AutomationFrequencySchema = z.enum(["disabled", "daily", "weekly"]); +export type AutomationFrequency = z.infer; + +export const AutomationScheduleSchema = z.object({ + frequency: AutomationFrequencySchema, + time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).default("03:00"), + dayOfWeek: z.number().int().min(0).max(6).default(1) +}); +export type AutomationScheduleDto = z.infer; + +export const AutomationSettingsSchema = z.object({ + watchLibraries: z.boolean(), + autoEnrichNewBooks: z.boolean(), + scanSchedule: AutomationScheduleSchema, + enrichSchedule: AutomationScheduleSchema +}); +export type AutomationSettingsDto = z.infer; + +export const UpdateAutomationSettingsSchema = AutomationSettingsSchema.partial(); +export type UpdateAutomationSettingsDto = z.infer;