From 8b1daea23c44aace402156d2f3b5978b0d7022a9 Mon Sep 17 00:00:00 2001 From: blomios Date: Thu, 8 Oct 2026 17:17:16 +0200 Subject: [PATCH] fix(live): bound live channel listing queries Optimize the live channel page path so unfiltered catalog pages are counted and paged in SQLite before serialization and enrichment. Add the T42 CPU regression test covering a 50000-channel projected catalog. Refs #42 Co-Authored-By: Claude Opus 4.8 --- server/db/liveProjectionSchema.js | 2 + server/services/liveProjectionService.js | 114 +++++++++++++------ test/t42LiveListCpuRegression.test.js | 136 +++++++++++++++++++++++ 3 files changed, 219 insertions(+), 33 deletions(-) create mode 100644 test/t42LiveListCpuRegression.test.js diff --git a/server/db/liveProjectionSchema.js b/server/db/liveProjectionSchema.js index f65fada..d596781 100644 --- a/server/db/liveProjectionSchema.js +++ b/server/db/liveProjectionSchema.js @@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = { const LIVE_PROJECTION_SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id); CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint); + CREATE INDEX IF NOT EXISTS idx_items_live_projection_page + ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank); CREATE TABLE IF NOT EXISTS logical_channels ( id TEXT PRIMARY KEY, diff --git a/server/services/liveProjectionService.js b/server/services/liveProjectionService.js index 22741f3..933fceb 100644 --- a/server/services/liveProjectionService.js +++ b/server/services/liveProjectionService.js @@ -506,42 +506,92 @@ function listChannel(channel) { function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) { database ||= defaultDatabase(); const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog; - const params = []; - let sourceFilter = ''; - if (sourceId !== null) { - sourceFilter = 'AND p.source_id = ?'; - params.push(sourceId); - } - const rows = database.prepare(` + const bounds = pageBounds(page, limit); + const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?'; + const sourceParams = sourceId === null ? [] : [sourceId]; + const visibilityFilter = includeHidden + ? '' + : 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0'; + const languageCodes = [...new Set(languages.filter(Boolean))]; + const languageFilter = languageCodes.length + ? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})` + : ''; + const filteredParams = [...sourceParams, ...languageCodes]; + + const availableLanguageCodes = database.prepare(` + SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code + FROM playlist_items p + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} + `).all(...sourceParams).map(row => row.language_code); + const availableLanguages = availableLanguageCodes.map(code => ({ + code, + short: code === 'und' ? 'UND' : code.toUpperCase(), + label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code) + })).sort((left, right) => left.label.localeCompare(right.label)); + + const selectRows = extraWhere => database.prepare(` SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite, - l.display_name, l.preferred_variant_id + l.display_name, l.preferred_variant_id, c.name AS category_name, + COALESCE(c.is_hidden, 0) AS category_hidden FROM playlist_items p JOIN logical_channels l ON l.id = p.logical_channel_id + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id AND f.item_id = p.item_id AND f.item_type = 'channel' - WHERE p.type = 'live' ${sourceFilter} - ORDER BY l.display_name, p.variant_rank - `).all(userId ?? -1, ...params); - const categoryRows = sourceId === null - ? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all() - : database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId); - const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category])); - for (const row of rows) { - const category = categories.get(`${row.source_id}\0${row.category_id}`); - row.category_name = category?.name || null; - row.category_hidden = category?.is_hidden || 0; + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere} + ORDER BY l.display_name, p.logical_channel_id, + COALESCE(p.language_code, 'und'), p.variant_rank, p.id + `); + + let filtered; + let total; + if (String(q).trim()) { + // matchesSearch also examines variant JSON-derived fields, so retain its exact + // semantics for searches while still applying safe SQL filters first. + const rows = selectRows('').all(userId ?? -1, ...filteredParams); + filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q)); + total = filtered.length; + filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit); + } else { + const groupedSql = ` + FROM playlist_items p + JOIN logical_channels l ON l.id = p.logical_channel_id + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} + GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`; + total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`) + .get(...filteredParams).total; + const keys = database.prepare(` + SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code, + MIN(l.display_name) AS display_name + ${groupedSql} + ORDER BY display_name, p.logical_channel_id, language_code + LIMIT ? OFFSET ? + `).all(...filteredParams, bounds.limit, bounds.offset); + if (!keys.length) { + filtered = []; + } else { + const keyFilter = `AND (${keys.map(() => + `(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)` + ).join(' OR ')})`; + const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]); + filtered = serializeChannels( + selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams), + includeHidden + ); + } } - const serialized = serializeChannels(rows, includeHidden); - const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()] - .map(language => ({ code: language.code, short: language.short, label: language.label })) - .sort((left, right) => left.label.localeCompare(right.label)); - const filtered = serialized.filter(channel => - (!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q) - ); - const persistedEpg = new Map(database.prepare(` + + const persistedEpg = filtered.length ? new Map(database.prepare(` SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data FROM live_epg_matches - `).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])); + WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')} + `).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code])) + .map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map(); for (const channel of filtered) { const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`); if (!match) continue; @@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null, language: channel.language.code }; } - const bounds = pageBounds(page, limit); - const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit); // Catalog loading must stay bounded: both enrichers perform synchronous SQLite work, // and EPG matching is especially expensive against large XMLTV catalogs. Enrich only // a genuinely targeted, bounded search; base projection data remains usable otherwise. - const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS; - const enriched = !shouldEnrich ? paged : paged.map(channel => { + const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS; + const enriched = !shouldEnrich ? filtered : filtered.map(channel => { const match = standardCatalog.findMatch(database, channel); if (match) { channel.standardName = match.name; @@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null, return { channels: enriched.map(listChannel), availableLanguages, - page: pageMetadata(filtered.length, bounds) + page: pageMetadata(total, bounds) }; } diff --git a/test/t42LiveListCpuRegression.test.js b/test/t42LiveListCpuRegression.test.js new file mode 100644 index 0000000..d45f554 --- /dev/null +++ b/test/t42LiveListCpuRegression.test.js @@ -0,0 +1,136 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { performance } = require('node:perf_hooks'); +const Database = require('better-sqlite3'); +const liveProjection = require('../server/services/liveProjectionService'); +const { + ensureColumns, + LIVE_PROJECTION_COLUMNS, + LIVE_PROJECTION_SCHEMA_SQL +} = require('../server/db/liveProjectionSchema'); + +function createProjectionDb() { + const db = new Database(':memory:'); + db.pragma('journal_mode = MEMORY'); + db.pragma('synchronous = OFF'); + db.exec(` + CREATE TABLE categories ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + category_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + parent_id TEXT, + is_hidden INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE playlist_items ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + category_id TEXT, + parent_id TEXT, + stream_icon TEXT, + stream_url TEXT, + container_extension TEXT, + rating REAL, + year TEXT, + added_at TEXT, + is_hidden INTEGER DEFAULT 0, + is_favorite INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + item_type TEXT NOT NULL + ); + `); + ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS); + db.exec(LIVE_PROJECTION_SCHEMA_SQL); + db.prepare(` + INSERT INTO categories (id, source_id, category_id, type, name) + VALUES ('cat:1', 1, 'general', 'live', 'General') + `).run(); + return db; +} + +function seedProjectedChannels(db, total) { + const logical = db.prepare(` + INSERT INTO logical_channels + (id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at) + VALUES (?, ?, ?, ?, 1, 1, 1) + `); + const item = db.prepare(` + INSERT INTO playlist_items + (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data, + logical_channel_id, grouping_fingerprint, grouping_input_hash, quality_label, quality_rank, + variant_rank, language_code, language_confidence, language_detector, language_input_hash) + VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, 'HD', 2, 0, 'en', 1, 'algo', ?) + `); + const insertAll = db.transaction(() => { + for (let index = 0; index < total; index++) { + const id = `1:${index}`; + const logicalId = `logical:${index}`; + const name = `Channel ${String(index).padStart(6, '0')}`; + logical.run(logicalId, `channel ${index}`, name, id); + item.run( + id, + String(index), + name, + `https://streams.test/${index}.m3u8`, + logicalId, + `channel ${index}`, + `group-hash-${index}`, + `language-hash-${index}` + ); + } + }); + insertAll(); +} + +test('/api/live/channels first page stays CPU-bounded for a large projected catalog', () => { + const db = createProjectionDb(); + seedProjectedChannels(db, 50000); + + try { + const cpuBefore = process.cpuUsage(); + const started = performance.now(); + const result = liveProjection.getChannelPage({ + includeHidden: false, + userId: 7, + page: 1, + limit: 100 + }, db, { + iptvOrgCatalog: { findMatch: () => null } + }); + const elapsedMs = performance.now() - started; + const cpu = process.cpuUsage(cpuBefore); + const metrics = { + rows: 50000, + returnedChannels: result.channels.length, + totalChannels: result.page.total, + elapsedMs: Math.round(elapsedMs), + cpuUserMs: Math.round(cpu.user / 1000), + cpuSystemMs: Math.round(cpu.system / 1000), + payloadBytes: Buffer.byteLength(JSON.stringify({ + filters: { availableLanguages: result.availableLanguages }, + channels: result.channels, + page: result.page + })) + }; + console.log('[T42 live list metrics]', JSON.stringify(metrics)); + + assert.equal(result.channels.length, 100); + assert.ok( + metrics.elapsedMs <= 500 && metrics.cpuUserMs <= 500, + `first live page should be bounded; measured ${JSON.stringify(metrics)}` + ); + } finally { + db.close(); + } +});