Merge branch 'feature/t42-live-list-cpu-regression' into develop (#42)

server/db/liveProjectionSchema.js

@@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = {
 const LIVE_PROJECTION_SCHEMA_SQL = `
     CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id);
     CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint);
+    CREATE INDEX IF NOT EXISTS idx_items_live_projection_page
+        ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank);
 
     CREATE TABLE IF NOT EXISTS logical_channels (
         id TEXT PRIMARY KEY,

server/services/liveProjectionService.js

@@ -506,42 +506,92 @@ function listChannel(channel) {
 function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) {
     database ||= defaultDatabase();
     const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog;
-    const params = [];
+    const bounds = pageBounds(page, limit);
-    let sourceFilter = '';
+    const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?';
-    if (sourceId !== null) {
+    const sourceParams = sourceId === null ? [] : [sourceId];
-        sourceFilter = 'AND p.source_id = ?';
+    const visibilityFilter = includeHidden
-        params.push(sourceId);
+        ? ''
-    }
+        : 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0';
-    const rows = database.prepare(`
+    const languageCodes = [...new Set(languages.filter(Boolean))];
+    const languageFilter = languageCodes.length
+        ? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})`
+        : '';
+    const filteredParams = [...sourceParams, ...languageCodes];
+
+    const availableLanguageCodes = database.prepare(`
+        SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code
+        FROM playlist_items p
+        LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
+            AND c.category_id = p.category_id
+        WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter}
+    `).all(...sourceParams).map(row => row.language_code);
+    const availableLanguages = availableLanguageCodes.map(code => ({
+        code,
+        short: code === 'und' ? 'UND' : code.toUpperCase(),
+        label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code)
+    })).sort((left, right) => left.label.localeCompare(right.label));
+
+    const selectRows = extraWhere => database.prepare(`
         SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
-            l.display_name, l.preferred_variant_id
+            l.display_name, l.preferred_variant_id, c.name AS category_name,
+            COALESCE(c.is_hidden, 0) AS category_hidden
         FROM playlist_items p
         JOIN logical_channels l ON l.id = p.logical_channel_id
+        LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
+            AND c.category_id = p.category_id
         LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
             AND f.item_id = p.item_id AND f.item_type = 'channel'
-        WHERE p.type = 'live' ${sourceFilter}
+        WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere}
-        ORDER BY l.display_name, p.variant_rank
+        ORDER BY l.display_name, p.logical_channel_id,
-    `).all(userId ?? -1, ...params);
+            COALESCE(p.language_code, 'und'), p.variant_rank, p.id
-    const categoryRows = sourceId === null
+    `);
-        ? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all()
+
-        : database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId);
+    let filtered;
-    const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category]));
+    let total;
-    for (const row of rows) {
+    if (String(q).trim()) {
-        const category = categories.get(`${row.source_id}\0${row.category_id}`);
+        // matchesSearch also examines variant JSON-derived fields, so retain its exact
-        row.category_name = category?.name || null;
+        // semantics for searches while still applying safe SQL filters first.
-        row.category_hidden = category?.is_hidden || 0;
+        const rows = selectRows('').all(userId ?? -1, ...filteredParams);
+        filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q));
+        total = filtered.length;
+        filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
+    } else {
+        const groupedSql = `
+            FROM playlist_items p
+            JOIN logical_channels l ON l.id = p.logical_channel_id
+            LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
+                AND c.category_id = p.category_id
+            WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter}
+            GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`;
+        total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`)
+            .get(...filteredParams).total;
+        const keys = database.prepare(`
+            SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code,
+                MIN(l.display_name) AS display_name
+            ${groupedSql}
+            ORDER BY display_name, p.logical_channel_id, language_code
+            LIMIT ? OFFSET ?
+        `).all(...filteredParams, bounds.limit, bounds.offset);
+        if (!keys.length) {
+            filtered = [];
+        } else {
+            const keyFilter = `AND (${keys.map(() =>
+                `(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)`
+            ).join(' OR ')})`;
+            const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]);
+            filtered = serializeChannels(
+                selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams),
+                includeHidden
+            );
+        }
     }
-    const serialized = serializeChannels(rows, includeHidden);
+
-    const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()]
+    const persistedEpg = filtered.length ? new Map(database.prepare(`
-        .map(language => ({ code: language.code, short: language.short, label: language.label }))
-        .sort((left, right) => left.label.localeCompare(right.label));
-    const filtered = serialized.filter(channel =>
-        (!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
-    );
-    const persistedEpg = new Map(database.prepare(`
         SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data
         FROM live_epg_matches
-    `).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match]));
+        WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')}
+    `).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code]))
+        .map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map();
     for (const channel of filtered) {
         const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`);
         if (!match) continue;
@@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
             language: channel.language.code
         };
     }
-    const bounds = pageBounds(page, limit);
-    const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
     // Catalog loading must stay bounded: both enrichers perform synchronous SQLite work,
     // and EPG matching is especially expensive against large XMLTV catalogs. Enrich only
     // a genuinely targeted, bounded search; base projection data remains usable otherwise.
-    const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS;
+    const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS;
-    const enriched = !shouldEnrich ? paged : paged.map(channel => {
+    const enriched = !shouldEnrich ? filtered : filtered.map(channel => {
         const match = standardCatalog.findMatch(database, channel);
         if (match) {
             channel.standardName = match.name;
@@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
     return {
         channels: enriched.map(listChannel),
         availableLanguages,
-        page: pageMetadata(filtered.length, bounds)
+        page: pageMetadata(total, bounds)
     };
 }
 

test/t42LiveListCpuRegression.test.js

@@ -0,0 +1,136 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { performance } = require('node:perf_hooks');
+const Database = require('better-sqlite3');
+const liveProjection = require('../server/services/liveProjectionService');
+const {
+    ensureColumns,
+    LIVE_PROJECTION_COLUMNS,
+    LIVE_PROJECTION_SCHEMA_SQL
+} = require('../server/db/liveProjectionSchema');
+
+function createProjectionDb() {
+    const db = new Database(':memory:');
+    db.pragma('journal_mode = MEMORY');
+    db.pragma('synchronous = OFF');
+    db.exec(`
+        CREATE TABLE categories (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            category_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            parent_id TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE playlist_items (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            category_id TEXT,
+            parent_id TEXT,
+            stream_icon TEXT,
+            stream_url TEXT,
+            container_extension TEXT,
+            rating REAL,
+            year TEXT,
+            added_at TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            is_favorite INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE favorites (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            user_id INTEGER NOT NULL,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            item_type TEXT NOT NULL
+        );
+    `);
+    ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
+    db.exec(LIVE_PROJECTION_SCHEMA_SQL);
+    db.prepare(`
+        INSERT INTO categories (id, source_id, category_id, type, name)
+        VALUES ('cat:1', 1, 'general', 'live', 'General')
+    `).run();
+    return db;
+}
+
+function seedProjectedChannels(db, total) {
+    const logical = db.prepare(`
+        INSERT INTO logical_channels
+            (id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at)
+        VALUES (?, ?, ?, ?, 1, 1, 1)
+    `);
+    const item = db.prepare(`
+        INSERT INTO playlist_items
+            (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data,
+             logical_channel_id, grouping_fingerprint, grouping_input_hash, quality_label, quality_rank,
+             variant_rank, language_code, language_confidence, language_detector, language_input_hash)
+        VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, 'HD', 2, 0, 'en', 1, 'algo', ?)
+    `);
+    const insertAll = db.transaction(() => {
+        for (let index = 0; index < total; index++) {
+            const id = `1:${index}`;
+            const logicalId = `logical:${index}`;
+            const name = `Channel ${String(index).padStart(6, '0')}`;
+            logical.run(logicalId, `channel ${index}`, name, id);
+            item.run(
+                id,
+                String(index),
+                name,
+                `https://streams.test/${index}.m3u8`,
+                logicalId,
+                `channel ${index}`,
+                `group-hash-${index}`,
+                `language-hash-${index}`
+            );
+        }
+    });
+    insertAll();
+}
+
+test('/api/live/channels first page stays CPU-bounded for a large projected catalog', () => {
+    const db = createProjectionDb();
+    seedProjectedChannels(db, 50000);
+
+    try {
+        const cpuBefore = process.cpuUsage();
+        const started = performance.now();
+        const result = liveProjection.getChannelPage({
+            includeHidden: false,
+            userId: 7,
+            page: 1,
+            limit: 100
+        }, db, {
+            iptvOrgCatalog: { findMatch: () => null }
+        });
+        const elapsedMs = performance.now() - started;
+        const cpu = process.cpuUsage(cpuBefore);
+        const metrics = {
+            rows: 50000,
+            returnedChannels: result.channels.length,
+            totalChannels: result.page.total,
+            elapsedMs: Math.round(elapsedMs),
+            cpuUserMs: Math.round(cpu.user / 1000),
+            cpuSystemMs: Math.round(cpu.system / 1000),
+            payloadBytes: Buffer.byteLength(JSON.stringify({
+                filters: { availableLanguages: result.availableLanguages },
+                channels: result.channels,
+                page: result.page
+            }))
+        };
+        console.log('[T42 live list metrics]', JSON.stringify(metrics));
+
+        assert.equal(result.channels.length, 100);
+        assert.ok(
+            metrics.elapsedMs <= 500 && metrics.cpuUserMs <= 500,
+            `first live page should be bounded; measured ${JSON.stringify(metrics)}`
+        );
+    } finally {
+        db.close();
+    }
+});