feat(api,shared): métadonnées multi-providers et automatisation des scans/enrichissements

Chaîne de résolution metadata (local, Open Library, Google Books, BNF)
avec activation/priorité/clé API par provider, colonnes isbn13 et
identifiers sur les livres, et intégration au scanner pour compléter
métadonnées et jaquettes manquantes. Module automation: réglages
persistés, planifications scan/enrichissement et déclenchement manuel,
exposés via des endpoints admin.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Git Agent
2026-08-23 13:10:45 +02:00
parent 1ac4144d0e
commit 48e9459cf3
23 changed files with 1029 additions and 24 deletions

View File

@ -1,9 +1,13 @@
import { Body, Controller, Delete, Get, Param, Patch, Post, UseGuards } from "@nestjs/common";
import { Body, Controller, Delete, Get, Param, Patch, Post, Put, UseGuards } from "@nestjs/common";
import {
CreateLibraryDto,
CreateLibrarySchema,
CreateUserDto,
CreateUserSchema,
UpdateAutomationSettingsDto,
UpdateAutomationSettingsSchema,
UpdateMetadataSourcesConfigDto,
UpdateMetadataSourcesConfigSchema,
UpdateLibraryDto,
UpdateLibrarySchema,
UpdateUserDto,
@ -13,9 +17,11 @@ import { AuthGuard } from "../auth/auth.guard.js";
import { Roles } from "../auth/roles.decorator.js";
import { RolesGuard } from "../auth/roles.guard.js";
import { AuthService } from "../auth/auth.service.js";
import { AutomationService } from "../automation/automation.service.js";
import { ZodValidationPipe } from "../common/zod-validation.pipe.js";
import { JobsService } from "../jobs/jobs.service.js";
import { LibrariesService } from "../libraries/libraries.service.js";
import { MetadataService } from "../metadata/metadata.service.js";
import { ScannerService } from "../scanner/scanner.service.js";
@Controller("admin")
@ -26,7 +32,9 @@ export class AdminController {
private readonly auth: AuthService,
private readonly libraries: LibrariesService,
private readonly jobs: JobsService,
private readonly scanner: ScannerService
private readonly scanner: ScannerService,
private readonly metadata: MetadataService,
private readonly automation: AutomationService
) {}
@Get("users")
@ -80,4 +88,34 @@ export class AdminController {
listJobs() {
return this.jobs.list();
}
@Get("metadata-sources")
metadataSources() {
return this.metadata.getSourcesConfig();
}
@Put("metadata-sources")
updateMetadataSources(@Body(new ZodValidationPipe(UpdateMetadataSourcesConfigSchema)) body: UpdateMetadataSourcesConfigDto) {
return this.metadata.updateSourcesConfig(body);
}
@Get("automation")
automationSettings() {
return this.automation.getSettings();
}
@Put("automation")
updateAutomationSettings(@Body(new ZodValidationPipe(UpdateAutomationSettingsSchema)) body: UpdateAutomationSettingsDto) {
return this.automation.updateSettings(body);
}
@Post("automation/run-scan")
runAutomationScan() {
return this.automation.runScanNow();
}
@Post("automation/run-enrich")
runAutomationEnrich() {
return this.automation.runEnrichNow();
}
}

View File

@ -1,13 +1,15 @@
import { Module } from "@nestjs/common";
import { AuthModule } from "../auth/auth.module.js";
import { AutomationModule } from "../automation/automation.module.js";
import { DatabaseModule } from "../database/database.module.js";
import { JobsModule } from "../jobs/jobs.module.js";
import { LibrariesService } from "../libraries/libraries.service.js";
import { MetadataModule } from "../metadata/metadata.module.js";
import { ScannerModule } from "../scanner/scanner.module.js";
import { AdminController } from "./admin.controller.js";
@Module({
imports: [AuthModule, DatabaseModule, JobsModule, ScannerModule],
imports: [AuthModule, DatabaseModule, JobsModule, ScannerModule, MetadataModule, AutomationModule],
controllers: [AdminController],
providers: [LibrariesService]
})

View File

@ -1,5 +1,6 @@
import { Module } from "@nestjs/common";
import { AdminModule } from "./admin/admin.module.js";
import { AutomationModule } from "./automation/automation.module.js";
import { AuthModule } from "./auth/auth.module.js";
import { BooksModule } from "./books/books.module.js";
import { DatabaseModule } from "./database/database.module.js";
@ -8,7 +9,7 @@ import { ScannerModule } from "./scanner/scanner.module.js";
import { HealthController } from "./health.controller.js";
@Module({
imports: [DatabaseModule, AuthModule, AdminModule, BooksModule, ProgressModule, ScannerModule],
imports: [DatabaseModule, AuthModule, AdminModule, BooksModule, ProgressModule, ScannerModule, AutomationModule],
controllers: [HealthController]
})
export class AppModule {}

View File

@ -0,0 +1,11 @@
import { Module } from "@nestjs/common";
import { DatabaseModule } from "../database/database.module.js";
import { ScannerModule } from "../scanner/scanner.module.js";
import { AutomationService } from "./automation.service.js";
@Module({
imports: [DatabaseModule, ScannerModule],
providers: [AutomationService],
exports: [AutomationService]
})
export class AutomationModule {}

View File

@ -0,0 +1,14 @@
import { describe, expect, it } from "vitest";
import { nextRunAt } from "./automation.service.js";
describe("automation scheduling", () => {
it("computes the next daily run", () => {
const next = nextRunAt({ frequency: "daily", time: "03:30", dayOfWeek: 1 }, new Date("2026-08-23T02:00:00.000Z"));
expect(next.toISOString()).toBe("2026-08-23T03:30:00.000Z");
});
it("moves elapsed weekly runs to the next week", () => {
const next = nextRunAt({ frequency: "weekly", time: "03:30", dayOfWeek: 0 }, new Date("2026-08-23T04:00:00.000Z"));
expect(next.toISOString()).toBe("2026-08-30T03:30:00.000Z");
});
});

View File

@ -0,0 +1,209 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from "@nestjs/common";
import { Dirent, FSWatcher, readdirSync, watch } from "node:fs";
import { join } from "node:path";
import { eq } from "drizzle-orm";
import { AutomationScheduleDto, AutomationSettingsDto, UpdateAutomationSettingsDto } from "@readabook/shared";
import { DatabaseService } from "../database/database.service.js";
import { automationSettings, libraries } from "../database/schema.js";
import { ScannerService } from "../scanner/scanner.service.js";
type WatchEntry = {
watchers: FSWatcher[];
timer: NodeJS.Timeout | null;
};
const defaultSchedule: AutomationScheduleDto = { frequency: "disabled", time: "03:00", dayOfWeek: 1 };
@Injectable()
export class AutomationService implements OnModuleInit, OnModuleDestroy {
private readonly watchers = new Map<number, WatchEntry>();
private scanTimer: NodeJS.Timeout | null = null;
private enrichTimer: NodeJS.Timeout | null = null;
constructor(
private readonly database: DatabaseService,
private readonly scanner: ScannerService
) {}
onModuleInit(): void {
this.applyRuntimeSettings();
}
onModuleDestroy(): void {
this.stopWatchers();
this.clearSchedules();
}
getSettings(): AutomationSettingsDto {
const row = this.readRow();
return {
watchLibraries: row.watchLibraries,
autoEnrichNewBooks: row.autoEnrichNewBooks,
scanSchedule: parseSchedule(row.scanScheduleJson),
enrichSchedule: parseSchedule(row.enrichScheduleJson)
};
}
updateSettings(input: UpdateAutomationSettingsDto): AutomationSettingsDto {
const current = this.getSettings();
const next: AutomationSettingsDto = {
watchLibraries: input.watchLibraries ?? current.watchLibraries,
autoEnrichNewBooks: input.autoEnrichNewBooks ?? current.autoEnrichNewBooks,
scanSchedule: input.scanSchedule ? normalizeSchedule(input.scanSchedule) : current.scanSchedule,
enrichSchedule: input.enrichSchedule ? normalizeSchedule(input.enrichSchedule) : current.enrichSchedule
};
this.database.db
.update(automationSettings)
.set({
watchLibraries: next.watchLibraries,
autoEnrichNewBooks: next.autoEnrichNewBooks,
scanScheduleJson: JSON.stringify(next.scanSchedule),
enrichScheduleJson: JSON.stringify(next.enrichSchedule),
updatedAt: this.database.now()
})
.where(eq(automationSettings.id, 1))
.run();
this.applyRuntimeSettings();
return this.getSettings();
}
runScanNow() {
return this.scanner.enqueueAllLibrariesScan("Manual automation scan");
}
runEnrichNow() {
return this.scanner.enqueueMetadataEnrichment("Manual metadata enrichment");
}
private applyRuntimeSettings(): void {
const settings = this.getSettings();
settings.watchLibraries ? this.startWatchers() : this.stopWatchers();
this.configureSchedules(settings);
}
private startWatchers(): void {
const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all();
const enabledIds = new Set(enabledLibraries.map((library) => library.id));
for (const [id, entry] of this.watchers) {
if (!enabledIds.has(id)) {
entry.watchers.forEach((watcher) => watcher.close());
if (entry.timer) clearTimeout(entry.timer);
this.watchers.delete(id);
}
}
for (const library of enabledLibraries) {
if (this.watchers.has(library.id)) continue;
const watchers = watchLibraryDirs(library.path, (_event, filename) => {
if (filename && !isBookPath(String(filename))) return;
const current = this.watchers.get(library.id);
if (!current) return;
if (current.timer) clearTimeout(current.timer);
current.timer = setTimeout(() => {
current.timer = null;
this.scanner.enqueueLibraryScan(library.id);
}, 1500);
});
for (const watcher of watchers) {
watcher.on("error", () => {
this.watchers.delete(library.id);
});
}
this.watchers.set(library.id, { watchers, timer: null });
}
}
private stopWatchers(): void {
for (const entry of this.watchers.values()) {
entry.watchers.forEach((watcher) => watcher.close());
if (entry.timer) clearTimeout(entry.timer);
}
this.watchers.clear();
}
private configureSchedules(settings: AutomationSettingsDto): void {
this.clearSchedules();
this.scanTimer = scheduleNext(settings.scanSchedule, () => {
this.scanner.enqueueAllLibrariesScan("Scheduled library scan");
this.configureSchedules(this.getSettings());
});
this.enrichTimer = scheduleNext(settings.enrichSchedule, () => {
this.scanner.enqueueMetadataEnrichment("Scheduled metadata enrichment");
this.configureSchedules(this.getSettings());
});
}
private clearSchedules(): void {
if (this.scanTimer) clearTimeout(this.scanTimer);
if (this.enrichTimer) clearTimeout(this.enrichTimer);
this.scanTimer = null;
this.enrichTimer = null;
}
private readRow(): typeof automationSettings.$inferSelect {
return this.database.db.select().from(automationSettings).where(eq(automationSettings.id, 1)).get()!;
}
}
export function scheduleNext(schedule: AutomationScheduleDto, run: () => void, now = new Date()): NodeJS.Timeout | null {
if (schedule.frequency === "disabled") return null;
const next = nextRunAt(schedule, now);
return setTimeout(run, Math.max(1000, next.getTime() - now.getTime()));
}
export function nextRunAt(schedule: AutomationScheduleDto, now = new Date()): Date {
const [hour, minute] = schedule.time.split(":").map(Number);
const next = new Date(now);
next.setUTCHours(hour, minute, 0, 0);
if (schedule.frequency === "weekly") {
const delta = (schedule.dayOfWeek - next.getUTCDay() + 7) % 7;
next.setUTCDate(next.getUTCDate() + delta);
}
if (next <= now) {
next.setUTCDate(next.getUTCDate() + (schedule.frequency === "weekly" ? 7 : 1));
}
return next;
}
function parseSchedule(value: string): AutomationScheduleDto {
try {
return normalizeSchedule(JSON.parse(value) as Partial<AutomationScheduleDto>);
} catch {
return defaultSchedule;
}
}
function normalizeSchedule(value: Partial<AutomationScheduleDto>): AutomationScheduleDto {
return {
frequency: value.frequency ?? "disabled",
time: value.time ?? "03:00",
dayOfWeek: value.dayOfWeek ?? 1
};
}
function isBookPath(filePath: string): boolean {
return /\.(epub|pdf|cbz|cbr)$/i.test(filePath);
}
function watchLibraryDirs(root: string, listener: (event: string, filename: string | Buffer | null) => void): FSWatcher[] {
const watchers: FSWatcher[] = [];
for (const dir of walkDirs(root)) {
watchers.push(watch(dir, listener));
}
return watchers;
}
function* walkDirs(root: string): Generator<string> {
yield root;
let entries: Dirent[];
try {
entries = readdirSync(root, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
if (entry.isDirectory()) {
yield* walkDirs(join(root, entry.name));
}
}
}

View File

@ -125,6 +125,7 @@ function mapBookRow(row: Record<string, unknown>) {
author: nullable(row.author),
description: nullable(row.description),
isbn: nullable(row.isbn),
isbn13: nullable(row.isbn13),
language: nullable(row.language),
publisher: nullable(row.publisher),
publishedDate: nullable(row.published_date),

View File

@ -56,6 +56,8 @@ export class DatabaseService implements OnModuleDestroy {
author TEXT,
description TEXT,
isbn TEXT,
isbn13 TEXT,
identifiers_json TEXT,
language TEXT,
publisher TEXT,
published_date TEXT,
@ -89,6 +91,26 @@ export class DatabaseService implements OnModuleDestroy {
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS metadata_source_config (
provider TEXT PRIMARY KEY CHECK (provider IN ('local','openlibrary','googlebooks','bnf')),
enabled INTEGER NOT NULL DEFAULT 1,
priority INTEGER NOT NULL,
api_key TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS automation_settings (
id INTEGER PRIMARY KEY CHECK (id = 1),
watch_libraries INTEGER NOT NULL DEFAULT 0,
auto_enrich_new_books INTEGER NOT NULL DEFAULT 1,
isbn_priority_enabled INTEGER NOT NULL DEFAULT 1,
scan_schedule_json TEXT NOT NULL,
enrich_schedule_json TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE VIRTUAL TABLE IF NOT EXISTS book_fts USING fts5(
title,
author,
@ -100,6 +122,7 @@ export class DatabaseService implements OnModuleDestroy {
CREATE INDEX IF NOT EXISTS books_library_idx ON books(library_id);
CREATE INDEX IF NOT EXISTS books_title_idx ON books(title);
CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13);
CREATE INDEX IF NOT EXISTS jobs_status_idx ON jobs(status);
CREATE TRIGGER IF NOT EXISTS books_ai AFTER INSERT ON books BEGIN
@ -120,6 +143,8 @@ export class DatabaseService implements OnModuleDestroy {
END;
`);
this.ensureBooksSupportsComicArchives();
this.ensureBooksMetadataColumns();
this.ensureMetadataDefaults();
this.sqlite.exec("INSERT INTO book_fts(book_fts) VALUES('rebuild')");
}
@ -146,6 +171,8 @@ export class DatabaseService implements OnModuleDestroy {
author TEXT,
description TEXT,
isbn TEXT,
isbn13 TEXT,
identifiers_json TEXT,
language TEXT,
publisher TEXT,
published_date TEXT,
@ -158,11 +185,11 @@ export class DatabaseService implements OnModuleDestroy {
updated_at TEXT NOT NULL
);
INSERT INTO books (
id, library_id, title, author, description, isbn, language, publisher, published_date,
id, library_id, title, author, description, isbn, isbn13, identifiers_json, language, publisher, published_date,
format, file_path, cover_path, file_size, file_mtime, created_at, updated_at
)
SELECT
id, library_id, title, author, description, isbn, language, publisher, published_date,
id, library_id, title, author, description, isbn, NULL, NULL, language, publisher, published_date,
format, file_path, cover_path, file_size, file_mtime, created_at, updated_at
FROM books_legacy_format;
DROP TABLE books_legacy_format;
@ -174,6 +201,7 @@ export class DatabaseService implements OnModuleDestroy {
CREATE UNIQUE INDEX IF NOT EXISTS books_file_path_unique ON books(file_path);
CREATE INDEX IF NOT EXISTS books_library_idx ON books(library_id);
CREATE INDEX IF NOT EXISTS books_title_idx ON books(title);
CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13);
CREATE TRIGGER IF NOT EXISTS books_ai AFTER INSERT ON books BEGIN
INSERT INTO book_fts(rowid, title, author, description, isbn)
@ -193,4 +221,47 @@ export class DatabaseService implements OnModuleDestroy {
END;
`);
}
private ensureBooksMetadataColumns(): void {
const columns = this.sqlite.prepare("PRAGMA table_info(books)").all() as Array<{ name: string }>;
const names = new Set(columns.map((column) => column.name));
if (!names.has("isbn13")) {
this.sqlite.exec("ALTER TABLE books ADD COLUMN isbn13 TEXT");
}
if (!names.has("identifiers_json")) {
this.sqlite.exec("ALTER TABLE books ADD COLUMN identifiers_json TEXT");
}
this.sqlite.exec("CREATE INDEX IF NOT EXISTS books_isbn13_idx ON books(isbn13)");
}
private ensureMetadataDefaults(): void {
const now = this.now();
const insertSource = this.sqlite.prepare(`
INSERT INTO metadata_source_config (provider, enabled, priority, api_key, created_at, updated_at)
VALUES (?, ?, ?, NULL, ?, ?)
ON CONFLICT(provider) DO NOTHING
`);
insertSource.run("local", 1, 0, now, now);
insertSource.run("openlibrary", this.config.openLibraryEnabled ? 1 : 0, 1, now, now);
insertSource.run("googlebooks", 0, 2, now, now);
insertSource.run("bnf", 0, 3, now, now);
this.sqlite
.prepare(
`
INSERT INTO automation_settings (
id, watch_libraries, auto_enrich_new_books, isbn_priority_enabled,
scan_schedule_json, enrich_schedule_json, created_at, updated_at
)
VALUES (1, 0, 1, 1, ?, ?, ?, ?)
ON CONFLICT(id) DO NOTHING
`
)
.run(
JSON.stringify({ frequency: "disabled", time: "03:00", dayOfWeek: 1 }),
JSON.stringify({ frequency: "disabled", time: "04:00", dayOfWeek: 1 }),
now,
now
);
}
}

View File

@ -34,6 +34,8 @@ export const books = sqliteTable(
author: text("author"),
description: text("description"),
isbn: text("isbn"),
isbn13: text("isbn13"),
identifiersJson: text("identifiers_json"),
language: text("language"),
publisher: text("publisher"),
publishedDate: text("published_date"),
@ -75,3 +77,23 @@ export const jobs = sqliteTable("jobs", {
createdAt: text("created_at").notNull(),
updatedAt: text("updated_at").notNull()
});
export const metadataSourceConfig = sqliteTable("metadata_source_config", {
provider: text("provider", { enum: ["local", "openlibrary", "googlebooks", "bnf"] }).primaryKey(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
priority: integer("priority").notNull(),
apiKey: text("api_key"),
createdAt: text("created_at").notNull(),
updatedAt: text("updated_at").notNull()
});
export const automationSettings = sqliteTable("automation_settings", {
id: integer("id").primaryKey(),
watchLibraries: integer("watch_libraries", { mode: "boolean" }).notNull().default(false),
autoEnrichNewBooks: integer("auto_enrich_new_books", { mode: "boolean" }).notNull().default(true),
isbnPriorityEnabled: integer("isbn_priority_enabled", { mode: "boolean" }).notNull().default(true),
scanScheduleJson: text("scan_schedule_json").notNull(),
enrichScheduleJson: text("enrich_schedule_json").notNull(),
createdAt: text("created_at").notNull(),
updatedAt: text("updated_at").notNull()
});

View File

@ -0,0 +1,38 @@
import { Injectable } from "@nestjs/common";
import { XMLParser } from "fast-xml-parser";
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" });
@Injectable()
export class BnfProvider implements MetadataProvider {
readonly id = "bnf" as const;
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch | null> {
const query = lookup.identifiers.isbn13
? `bib.isbn all "${lookup.identifiers.isbn13}"`
: `bib.title all "${lookup.title.replace(/"/g, " ")}"`;
const url = new URL("https://catalogue.bnf.fr/api/SRU");
url.searchParams.set("version", "1.2");
url.searchParams.set("operation", "searchRetrieve");
url.searchParams.set("query", query);
url.searchParams.set("maximumRecords", "1");
const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
if (!response.ok) return null;
const parsed = parser.parse(await response.text());
const record = parsed?.searchRetrieveResponse?.records?.record?.recordData;
if (!record) return null;
const text = JSON.stringify(record);
return {
title: match(text, /"titleInfo"[^}]*"title":"([^"]+)"/),
author: match(text, /"namePart":"([^"]+)"/),
publisher: match(text, /"publisher":"([^"]+)"/),
publishedDate: match(text, /"dateIssued":"([^"]+)"/)
};
}
}
function match(value: string, pattern: RegExp): string | undefined {
return value.match(pattern)?.[1];
}

View File

@ -0,0 +1,47 @@
import { Injectable } from "@nestjs/common";
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
@Injectable()
export class GoogleBooksProvider implements MetadataProvider {
readonly id = "googlebooks" as const;
async lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null> {
const query = lookup.identifiers.isbn13
? `isbn:${lookup.identifiers.isbn13}`
: `intitle:${lookup.title}${lookup.author ? `+inauthor:${lookup.author}` : ""}`;
const url = new URL("https://www.googleapis.com/books/v1/volumes");
url.searchParams.set("q", query);
url.searchParams.set("maxResults", "1");
if (config.apiKey) url.searchParams.set("key", config.apiKey);
const response = await fetch(url, { signal: AbortSignal.timeout(4000) });
if (!response.ok) return null;
const data = (await response.json()) as { items?: Array<{ volumeInfo?: Record<string, unknown> }> };
const info = data.items?.[0]?.volumeInfo;
if (!info) return null;
return {
title: stringValue(info.title) ?? undefined,
author: arrayJoin(info.authors),
description: stringValue(info.description),
language: stringValue(info.language),
publisher: stringValue(info.publisher),
publishedDate: stringValue(info.publishedDate),
isbn: isbnFromIndustryIdentifiers(info.industryIdentifiers)
};
}
}
function stringValue(value: unknown): string | null {
return typeof value === "string" && value.trim() ? value.trim() : null;
}
function arrayJoin(value: unknown): string | null {
return Array.isArray(value) && value.length ? value.map(String).join(", ") : null;
}
function isbnFromIndustryIdentifiers(value: unknown): string | null {
if (!Array.isArray(value)) return null;
const isbn13 = value.find((entry) => entry?.type === "ISBN_13")?.identifier;
const isbn10 = value.find((entry) => entry?.type === "ISBN_10")?.identifier;
return stringValue(isbn13) ?? stringValue(isbn10);
}

View File

@ -0,0 +1,16 @@
import { Injectable } from "@nestjs/common";
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
@Injectable()
export class LocalMetadataProvider implements MetadataProvider {
readonly id = "local" as const;
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch> {
return {
title: lookup.title,
author: lookup.author,
isbn: lookup.identifiers.isbn13 ?? lookup.identifiers.isbn10,
identifiers: lookup.identifiers
};
}
}

View File

@ -0,0 +1,33 @@
import { Injectable } from "@nestjs/common";
import { MetadataLookup, MetadataMatch, MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
@Injectable()
export class OpenLibraryProvider implements MetadataProvider {
readonly id = "openlibrary" as const;
async lookup(lookup: MetadataLookup, _config: MetadataProviderConfig): Promise<MetadataMatch | null> {
const query = lookup.identifiers.isbn13
? `isbn:${encodeURIComponent(lookup.identifiers.isbn13)}`
: `title:${encodeURIComponent(lookup.title)}${lookup.author ? ` author:${encodeURIComponent(lookup.author)}` : ""}`;
const response = await fetch(`https://openlibrary.org/search.json?q=${query}&limit=1`, {
headers: { "User-Agent": "ReadaBook/0.1 local-library-manager" },
signal: AbortSignal.timeout(4000)
});
if (!response.ok) return null;
const data = (await response.json()) as { docs?: Array<Record<string, unknown>> };
const doc = data.docs?.[0];
if (!doc) return null;
return {
author: firstArrayValue(doc.author_name),
language: firstArrayValue(doc.language),
publisher: firstArrayValue(doc.publisher),
publishedDate: String(doc.first_publish_year ?? "") || null,
isbn: firstArrayValue(doc.isbn)
};
}
}
function firstArrayValue(value: unknown): string | null {
if (!Array.isArray(value) || !value.length) return null;
return String(value[0]);
}

View File

@ -0,0 +1,31 @@
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { ExtractIdentifiers, normalizeIsbn, toIsbn13 } from "./use-cases/extract-identifiers.js";
describe("ISBN normalization", () => {
it("accepts valid ISBN-10/13 and rejects bad checksums", () => {
expect(normalizeIsbn("0-306-40615-2")).toBe("0306406152");
expect(normalizeIsbn("978-0-306-40615-7")).toBe("9780306406157");
expect(normalizeIsbn("978-0-306-40615-8")).toBeNull();
});
it("converts ISBN-10 to ISBN-13", () => {
expect(toIsbn13("0-306-40615-2")).toBe("9780306406157");
});
});
describe("ExtractIdentifiers", () => {
it("uses embedded PDF text before filename fallback candidates", () => {
const dir = mkdtempSync(join(tmpdir(), "readabook-isbn-"));
const file = join(dir, "fallback 9780306406157.pdf");
writeFileSync(file, "%PDF-1.4\n1 0 obj << /Title (Book) /Subject (ISBN 0-306-40615-2) >> endobj");
const identifiers = new ExtractIdentifiers().fromMetadataAndFile({ isbn: null }, file);
expect(identifiers.isbn10).toBe("0306406152");
expect(identifiers.isbn13).toBe("9780306406157");
expect(identifiers.candidates).toContain("9780306406157");
});
});

View File

@ -0,0 +1,14 @@
import { Module } from "@nestjs/common";
import { DatabaseModule } from "../database/database.module.js";
import { BnfProvider } from "./adapters/bnf.provider.js";
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
import { LocalMetadataProvider } from "./adapters/local.provider.js";
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
import { MetadataService } from "./metadata.service.js";
@Module({
imports: [DatabaseModule],
providers: [MetadataService, LocalMetadataProvider, OpenLibraryProvider, GoogleBooksProvider, BnfProvider],
exports: [MetadataService]
})
export class MetadataModule {}

View File

@ -0,0 +1,167 @@
import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
import { eq } from "drizzle-orm";
import {
MetadataSourcesConfigDto,
UpdateMetadataSourcesConfigDto
} from "@readabook/shared";
import { DatabaseService } from "../database/database.service.js";
import { automationSettings, books, metadataSourceConfig } from "../database/schema.js";
import { BookMetadata } from "../scanner/metadata.js";
import { BnfProvider } from "./adapters/bnf.provider.js";
import { GoogleBooksProvider } from "./adapters/google-books.provider.js";
import { LocalMetadataProvider } from "./adapters/local.provider.js";
import { OpenLibraryProvider } from "./adapters/open-library.provider.js";
import { MetadataMatch, MetadataProvider, MetadataProviderConfig } from "./metadata.types.js";
import { ExtractIdentifiers, toIsbn13 } from "./use-cases/extract-identifiers.js";
import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js";
@Injectable()
export class MetadataService {
private readonly extractIdentifiers = new ExtractIdentifiers();
private readonly resolveProviderChain: ResolveProviderChain;
constructor(
private readonly database: DatabaseService,
local: LocalMetadataProvider,
openLibrary: OpenLibraryProvider,
googleBooks: GoogleBooksProvider,
bnf: BnfProvider
) {
this.resolveProviderChain = new ResolveProviderChain([local, openLibrary, googleBooks, bnf]);
}
getSourcesConfig(): MetadataSourcesConfigDto {
const settings = this.getAutomationRow();
return {
isbnPriorityEnabled: Boolean(settings.isbnPriorityEnabled),
sources: this.getProviderConfigs().map((source) => ({
provider: source.provider,
enabled: source.provider === "local" ? true : source.enabled,
priority: source.priority,
hasApiKey: Boolean(source.apiKey)
}))
};
}
updateSourcesConfig(input: UpdateMetadataSourcesConfigDto): MetadataSourcesConfigDto {
const now = this.database.now();
if (input.isbnPriorityEnabled !== undefined) {
this.database.db
.update(automationSettings)
.set({ isbnPriorityEnabled: input.isbnPriorityEnabled, updatedAt: now })
.where(eq(automationSettings.id, 1))
.run();
}
for (const source of input.sources ?? []) {
if ((source.provider as string) === "local") {
throw new BadRequestException("Local metadata source is always active and cannot be updated");
}
const values: Partial<typeof metadataSourceConfig.$inferInsert> = {
enabled: source.enabled,
priority: source.priority,
updatedAt: now
};
if (source.apiKey !== undefined) values.apiKey = source.apiKey;
this.database.db.update(metadataSourceConfig).set(values).where(eq(metadataSourceConfig.provider, source.provider)).run();
}
return this.getSourcesConfig();
}
async enrichMetadata(localMetadata: BookMetadata, filePath: string, options: { remote: boolean }): Promise<BookMetadata & { isbn13: string | null; identifiersJson: string }> {
const identifiers = this.extractIdentifiers.fromMetadataAndFile(localMetadata, filePath);
const configs = this.getProviderConfigs();
const chain = options.remote
? this.resolveProviderChain.resolve(configs)
: this.resolveProviderChain.resolve(configs).filter((entry) => entry.provider.id === "local");
let merged: BookMetadata = { ...localMetadata };
for (const { provider, config } of chain) {
try {
const match = await provider.lookup(
{
title: merged.title,
author: merged.author,
filePath,
identifiers
},
config
);
if (match) merged = mergeMetadata(merged, match);
} catch {
// Provider failures must not block local ingestion.
}
}
const isbn13 = identifiers.isbn13 ?? (merged.isbn ? toIsbn13(merged.isbn) : null);
return {
...merged,
isbn: merged.isbn ?? isbn13 ?? identifiers.isbn10,
isbn13,
identifiersJson: JSON.stringify(identifiers)
};
}
async enrichBook(bookId: number): Promise<typeof books.$inferSelect> {
const book = this.database.db.select().from(books).where(eq(books.id, bookId)).get();
if (!book) throw new NotFoundException("Book not found");
const metadata: BookMetadata = {
title: book.title,
author: book.author,
description: book.description,
isbn: book.isbn,
language: book.language,
publisher: book.publisher,
publishedDate: book.publishedDate,
coverPath: book.coverPath
};
const enriched = await this.enrichMetadata(metadata, book.filePath, { remote: true });
return this.database.db
.update(books)
.set({
title: enriched.title,
author: enriched.author,
description: enriched.description,
isbn: enriched.isbn,
isbn13: enriched.isbn13,
identifiersJson: enriched.identifiersJson,
language: enriched.language,
publisher: enriched.publisher,
publishedDate: enriched.publishedDate,
coverPath: enriched.coverPath,
updatedAt: this.database.now()
})
.where(eq(books.id, book.id))
.returning()
.get();
}
private getProviderConfigs(): MetadataProviderConfig[] {
return this.database.db
.select()
.from(metadataSourceConfig)
.all()
.map((row) => ({
provider: row.provider,
enabled: row.provider === "local" ? true : row.enabled,
priority: row.provider === "local" ? 0 : row.priority,
apiKey: row.apiKey
}));
}
private getAutomationRow(): typeof automationSettings.$inferSelect {
return this.database.db.select().from(automationSettings).where(eq(automationSettings.id, 1)).get()!;
}
}
function mergeMetadata(current: BookMetadata, next: MetadataMatch): BookMetadata {
return {
title: next.title ?? current.title,
author: current.author ?? next.author ?? null,
description: current.description ?? next.description ?? null,
isbn: current.isbn ?? next.isbn ?? null,
language: current.language ?? next.language ?? null,
publisher: current.publisher ?? next.publisher ?? null,
publishedDate: current.publishedDate ?? next.publishedDate ?? null,
coverPath: current.coverPath ?? next.coverPath ?? null
};
}

View File

@ -0,0 +1,32 @@
import { BookMetadata } from "../scanner/metadata.js";
export type MetadataProviderId = "local" | "openlibrary" | "googlebooks" | "bnf";
export type BookIdentifiers = {
isbn10: string | null;
isbn13: string | null;
candidates: string[];
};
export type MetadataLookup = {
title: string;
author: string | null;
filePath: string;
identifiers: BookIdentifiers;
};
export type MetadataMatch = Partial<BookMetadata> & {
identifiers?: Partial<BookIdentifiers>;
};
export type MetadataProviderConfig = {
provider: MetadataProviderId;
enabled: boolean;
priority: number;
apiKey: string | null;
};
export interface MetadataProvider {
readonly id: MetadataProviderId;
lookup(lookup: MetadataLookup, config: MetadataProviderConfig): Promise<MetadataMatch | null>;
}

View File

@ -0,0 +1,20 @@
import { describe, expect, it } from "vitest";
import { MetadataProvider } from "./metadata.types.js";
import { ResolveProviderChain } from "./use-cases/resolve-provider-chain.js";
const provider = (id: MetadataProvider["id"]): MetadataProvider => ({
id,
lookup: async () => null
});
describe("ResolveProviderChain", () => {
it("keeps local active and orders enabled remote providers by priority", () => {
const chain = new ResolveProviderChain([provider("googlebooks"), provider("local"), provider("bnf")]).resolve([
{ provider: "local", enabled: false, priority: 99, apiKey: null },
{ provider: "googlebooks", enabled: true, priority: 2, apiKey: null },
{ provider: "bnf", enabled: true, priority: 1, apiKey: null }
]);
expect(chain.map((entry) => entry.provider.id)).toEqual(["bnf", "googlebooks", "local"]);
});
});

View File

@ -0,0 +1,114 @@
import { readFileSync } from "node:fs";
import { basename, extname } from "node:path";
import AdmZip from "adm-zip";
import { XMLParser } from "fast-xml-parser";
export type ExtractedIdentifiers = {
isbn10: string | null;
isbn13: string | null;
candidates: string[];
};
const xmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: "@_",
textNodeName: "#text"
});
export class ExtractIdentifiers {
fromMetadataAndFile(metadata: { isbn?: string | null }, filePath: string): ExtractedIdentifiers {
const candidates = new Set<string>();
for (const value of [metadata.isbn, basename(filePath, extname(filePath))]) {
for (const isbn of findIsbns(String(value ?? ""))) candidates.add(isbn);
}
const extension = extname(filePath).toLowerCase();
if (extension === ".epub") {
for (const isbn of findIsbns(readLimitedEpubText(filePath))) candidates.add(isbn);
}
if (extension === ".pdf") {
const buffer = readFileSync(filePath);
const head = buffer.subarray(0, Math.min(buffer.length, 256 * 1024)).toString("latin1");
for (const isbn of findIsbns(head)) candidates.add(isbn);
}
return normalizeCandidates([...candidates]);
}
}
export function normalizeIsbn(value: string): string | null {
const compact = value.replace(/[^0-9X]/gi, "").toUpperCase();
if (compact.length === 10 && isValidIsbn10(compact)) return compact;
if (compact.length === 13 && isValidIsbn13(compact)) return compact;
return null;
}
export function toIsbn13(value: string): string | null {
const isbn = normalizeIsbn(value);
if (!isbn) return null;
if (isbn.length === 13) return isbn;
const stem = `978${isbn.slice(0, 9)}`;
let sum = 0;
for (let index = 0; index < stem.length; index += 1) {
sum += Number(stem[index]) * (index % 2 === 0 ? 1 : 3);
}
return `${stem}${(10 - (sum % 10)) % 10}`;
}
function normalizeCandidates(values: string[]): ExtractedIdentifiers {
const normalized = [...new Set(values.map(normalizeIsbn).filter((value): value is string => Boolean(value)))];
const isbn13 = normalized.map(toIsbn13).find((value): value is string => Boolean(value)) ?? null;
const isbn10 = normalized.find((value) => value.length === 10) ?? null;
return { isbn10, isbn13, candidates: normalized };
}
function findIsbns(text: string): string[] {
const matches = text.match(/(?:ISBN(?:-1[03])?:?\s*)?(?:97[89][-\s]?)?(?:\d[-\s]?){9,12}[\dX]/gi) ?? [];
return matches.map((match) => match.replace(/^ISBN(?:-1[03])?:?\s*/i, ""));
}
function isValidIsbn10(value: string): boolean {
let sum = 0;
for (let index = 0; index < 10; index += 1) {
const char = value[index];
const digit = char === "X" && index === 9 ? 10 : Number(char);
if (!Number.isInteger(digit)) return false;
sum += digit * (10 - index);
}
return sum % 11 === 0;
}
function isValidIsbn13(value: string): boolean {
let sum = 0;
for (let index = 0; index < 13; index += 1) {
const digit = Number(value[index]);
if (!Number.isInteger(digit)) return false;
sum += digit * (index % 2 === 0 ? 1 : 3);
}
return sum % 10 === 0;
}
function readLimitedEpubText(filePath: string): string {
try {
const zip = new AdmZip(filePath);
const fragments: string[] = [zip.readAsText("META-INF/container.xml")];
for (const entry of zip.getEntries()) {
if (fragments.join("").length > 256 * 1024) break;
if (!entry.isDirectory && /\.(opf|xhtml|html|htm|xml)$/i.test(entry.entryName)) {
fragments.push(stripXml(zip.readAsText(entry)));
}
}
return fragments.join("\n");
} catch {
return "";
}
}
function stripXml(value: string): string {
try {
const parsed = xmlParser.parse(value);
return JSON.stringify(parsed).slice(0, 256 * 1024);
} catch {
return value.slice(0, 256 * 1024);
}
}

View File

@ -0,0 +1,14 @@
import { MetadataProvider, MetadataProviderConfig } from "../metadata.types.js";
export class ResolveProviderChain {
constructor(private readonly providers: MetadataProvider[]) {}
resolve(configs: MetadataProviderConfig[]): Array<{ provider: MetadataProvider; config: MetadataProviderConfig }> {
const configById = new Map(configs.map((config) => [config.provider, config]));
return this.providers
.map((provider) => ({ provider, config: configById.get(provider.id) }))
.filter((entry): entry is { provider: MetadataProvider; config: MetadataProviderConfig } => Boolean(entry.config))
.filter((entry) => entry.config.provider === "local" || entry.config.enabled)
.sort((left, right) => left.config.priority - right.config.priority);
}
}

View File

@ -1,12 +1,12 @@
import { Module } from "@nestjs/common";
import { DatabaseModule } from "../database/database.module.js";
import { JobsModule } from "../jobs/jobs.module.js";
import { OpenLibraryService } from "./open-library.service.js";
import { MetadataModule } from "../metadata/metadata.module.js";
import { ScannerService } from "./scanner.service.js";
@Module({
imports: [DatabaseModule, JobsModule],
providers: [ScannerService, OpenLibraryService],
imports: [DatabaseModule, JobsModule, MetadataModule],
providers: [ScannerService],
exports: [ScannerService]
})
export class ScannerModule {}

View File

@ -1,19 +1,19 @@
import { Injectable, NotFoundException } from "@nestjs/common";
import { readdirSync, statSync } from "node:fs";
import { existsSync, readdirSync, statSync } from "node:fs";
import { extname, join } from "node:path";
import { eq } from "drizzle-orm";
import { eq, inArray } from "drizzle-orm";
import { DatabaseService } from "../database/database.service.js";
import { books, libraries } from "../database/schema.js";
import { automationSettings, books, libraries } from "../database/schema.js";
import { JobsService } from "../jobs/jobs.service.js";
import { MetadataService } from "../metadata/metadata.service.js";
import { extractMetadata } from "./metadata.js";
import { OpenLibraryService } from "./open-library.service.js";
@Injectable()
export class ScannerService {
constructor(
private readonly database: DatabaseService,
private readonly jobs: JobsService,
private readonly openLibrary: OpenLibraryService
private readonly metadata: MetadataService
) {}
enqueueLibraryScan(libraryId: number) {
@ -28,36 +28,75 @@ export class ScannerService {
return job;
}
enqueueAllLibrariesScan(detail = "Scanning all enabled libraries") {
const job = this.jobs.create("library-scan-all", detail);
setImmediate(() => {
void this.scanAllLibraries(job.id).catch((error) => this.jobs.markFailed(job.id, error));
});
return job;
}
enqueueMetadataEnrichment(detail = "Enriching existing books") {
const job = this.jobs.create("metadata-enrich", detail);
setImmediate(() => {
void this.enrichExistingBooks(job.id).catch((error) => this.jobs.markFailed(job.id, error));
});
return job;
}
private async scanLibrary(jobId: number, library: typeof libraries.$inferSelect): Promise<void> {
this.jobs.markRunning(jobId, `Scanning ${library.path}`);
let count = 0;
const seen = new Set<string>();
for (const filePath of walkBooks(library.path)) {
seen.add(filePath);
await this.ingestFile(library.id, filePath);
count += 1;
}
this.jobs.markSucceeded(jobId, `Scanned ${count} file(s)`);
const removed = this.removeMissingBooks(library.id, seen);
this.jobs.markSucceeded(jobId, `Scanned ${count} file(s), removed ${removed} missing book(s)`);
}
private async scanAllLibraries(jobId: number): Promise<void> {
this.jobs.markRunning(jobId, "Scanning all enabled libraries");
const enabledLibraries = this.database.db.select().from(libraries).where(eq(libraries.enabled, true)).all();
let scanned = 0;
for (const library of enabledLibraries) {
for (const filePath of walkBooks(library.path)) {
await this.ingestFile(library.id, filePath);
scanned += 1;
}
}
this.jobs.markSucceeded(jobId, `Scanned ${scanned} file(s) across ${enabledLibraries.length} library/libraries`);
}
private async enrichExistingBooks(jobId: number): Promise<void> {
this.jobs.markRunning(jobId, "Enriching existing books");
const rows = this.database.db.select({ id: books.id }).from(books).all();
let count = 0;
for (const row of rows) {
await this.metadata.enrichBook(row.id);
count += 1;
}
this.jobs.markSucceeded(jobId, `Enriched ${count} book(s)`);
}
private async ingestFile(libraryId: number, filePath: string): Promise<void> {
const stats = statSync(filePath);
let metadata = await extractMetadata(filePath, this.database.config.storageDir);
if (this.database.config.openLibraryEnabled) {
try {
metadata = { ...metadata, ...(await this.openLibrary.enrich(metadata)) };
} catch {
// Remote enrichment is opportunistic; local ingestion must stay deterministic.
}
}
const localMetadata = await extractMetadata(filePath, this.database.config.storageDir);
const now = this.database.now();
const format = bookFormatFromPath(filePath);
const existing = this.database.db.select({ id: books.id }).from(books).where(eq(books.filePath, filePath)).get();
const shouldRemoteEnrich = !existing ? this.shouldAutoEnrichNewBooks() : true;
const metadata = await this.metadata.enrichMetadata(localMetadata, filePath, { remote: shouldRemoteEnrich });
const values = {
libraryId,
title: metadata.title,
author: metadata.author,
description: metadata.description,
isbn: metadata.isbn,
isbn13: metadata.isbn13,
identifiersJson: metadata.identifiersJson,
language: metadata.language,
publisher: metadata.publisher,
publishedDate: metadata.publishedDate,
@ -73,6 +112,24 @@ export class ScannerService {
? this.database.db.update(books).set(values).where(eq(books.id, existing.id)).returning().get()
: this.database.db.insert(books).values({ ...values, createdAt: now }).returning().get();
}
private removeMissingBooks(libraryId: number, seen: Set<string>): number {
const existing = this.database.db.select({ id: books.id, filePath: books.filePath }).from(books).where(eq(books.libraryId, libraryId)).all();
const missing = existing.filter((book) => !seen.has(book.filePath) && !existsSync(book.filePath));
if (!missing.length) return 0;
this.database.db.delete(books).where(inArray(books.id, missing.map((book) => book.id))).run();
return missing.length;
}
private shouldAutoEnrichNewBooks(): boolean {
return Boolean(
this.database.db
.select({ autoEnrichNewBooks: automationSettings.autoEnrichNewBooks })
.from(automationSettings)
.where(eq(automationSettings.id, 1))
.get()?.autoEnrichNewBooks
);
}
}
function* walkBooks(root: string): Generator<string> {

View File

@ -91,6 +91,7 @@ export const BookSchema = z.object({
author: z.string().nullable(),
description: z.string().nullable(),
isbn: z.string().nullable(),
isbn13: z.string().nullable(),
language: z.string().nullable(),
publisher: z.string().nullable(),
publishedDate: z.string().nullable(),
@ -137,3 +138,55 @@ export const JobSchema = z.object({
updatedAt: z.string()
});
export type JobDto = z.infer<typeof JobSchema>;
export const MetadataProviderIdSchema = z.enum(["local", "openlibrary", "googlebooks", "bnf"]);
export type MetadataProviderId = z.infer<typeof MetadataProviderIdSchema>;
export const MetadataSourceConfigSchema = z.object({
provider: MetadataProviderIdSchema,
enabled: z.boolean(),
priority: z.number().int().min(0),
hasApiKey: z.boolean().default(false)
});
export type MetadataSourceConfigDto = z.infer<typeof MetadataSourceConfigSchema>;
export const MetadataSourcesConfigSchema = z.object({
isbnPriorityEnabled: z.boolean(),
sources: z.array(MetadataSourceConfigSchema)
});
export type MetadataSourcesConfigDto = z.infer<typeof MetadataSourcesConfigSchema>;
export const UpdateMetadataSourceConfigSchema = z.object({
provider: MetadataProviderIdSchema.exclude(["local"]),
enabled: z.boolean(),
priority: z.number().int().min(1).max(100),
apiKey: z.string().min(1).nullable().optional()
});
export type UpdateMetadataSourceConfigDto = z.infer<typeof UpdateMetadataSourceConfigSchema>;
export const UpdateMetadataSourcesConfigSchema = z.object({
isbnPriorityEnabled: z.boolean().optional(),
sources: z.array(UpdateMetadataSourceConfigSchema).optional()
});
export type UpdateMetadataSourcesConfigDto = z.infer<typeof UpdateMetadataSourcesConfigSchema>;
export const AutomationFrequencySchema = z.enum(["disabled", "daily", "weekly"]);
export type AutomationFrequency = z.infer<typeof AutomationFrequencySchema>;
export const AutomationScheduleSchema = z.object({
frequency: AutomationFrequencySchema,
time: z.string().regex(/^([01]\d|2[0-3]):[0-5]\d$/).default("03:00"),
dayOfWeek: z.number().int().min(0).max(6).default(1)
});
export type AutomationScheduleDto = z.infer<typeof AutomationScheduleSchema>;
export const AutomationSettingsSchema = z.object({
watchLibraries: z.boolean(),
autoEnrichNewBooks: z.boolean(),
scanSchedule: AutomationScheduleSchema,
enrichSchedule: AutomationScheduleSchema
});
export type AutomationSettingsDto = z.infer<typeof AutomationSettingsSchema>;
export const UpdateAutomationSettingsSchema = AutomationSettingsSchema.partial();
export type UpdateAutomationSettingsDto = z.infer<typeof UpdateAutomationSettingsSchema>;