diff --git a/server/db/liveProjectionSchema.js b/server/db/liveProjectionSchema.js index f65fada..d596781 100644 --- a/server/db/liveProjectionSchema.js +++ b/server/db/liveProjectionSchema.js @@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = { const LIVE_PROJECTION_SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id); CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint); + CREATE INDEX IF NOT EXISTS idx_items_live_projection_page + ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank); CREATE TABLE IF NOT EXISTS logical_channels ( id TEXT PRIMARY KEY, diff --git a/server/services/liveProjectionService.js b/server/services/liveProjectionService.js index 22741f3..933fceb 100644 --- a/server/services/liveProjectionService.js +++ b/server/services/liveProjectionService.js @@ -506,42 +506,92 @@ function listChannel(channel) { function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) { database ||= defaultDatabase(); const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog; - const params = []; - let sourceFilter = ''; - if (sourceId !== null) { - sourceFilter = 'AND p.source_id = ?'; - params.push(sourceId); - } - const rows = database.prepare(` + const bounds = pageBounds(page, limit); + const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?'; + const sourceParams = sourceId === null ? [] : [sourceId]; + const visibilityFilter = includeHidden + ? '' + : 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0'; + const languageCodes = [...new Set(languages.filter(Boolean))]; + const languageFilter = languageCodes.length + ? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})` + : ''; + const filteredParams = [...sourceParams, ...languageCodes]; + + const availableLanguageCodes = database.prepare(` + SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code + FROM playlist_items p + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} + `).all(...sourceParams).map(row => row.language_code); + const availableLanguages = availableLanguageCodes.map(code => ({ + code, + short: code === 'und' ? 'UND' : code.toUpperCase(), + label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code) + })).sort((left, right) => left.label.localeCompare(right.label)); + + const selectRows = extraWhere => database.prepare(` SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite, - l.display_name, l.preferred_variant_id + l.display_name, l.preferred_variant_id, c.name AS category_name, + COALESCE(c.is_hidden, 0) AS category_hidden FROM playlist_items p JOIN logical_channels l ON l.id = p.logical_channel_id + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id AND f.item_id = p.item_id AND f.item_type = 'channel' - WHERE p.type = 'live' ${sourceFilter} - ORDER BY l.display_name, p.variant_rank - `).all(userId ?? -1, ...params); - const categoryRows = sourceId === null - ? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all() - : database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId); - const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category])); - for (const row of rows) { - const category = categories.get(`${row.source_id}\0${row.category_id}`); - row.category_name = category?.name || null; - row.category_hidden = category?.is_hidden || 0; + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere} + ORDER BY l.display_name, p.logical_channel_id, + COALESCE(p.language_code, 'und'), p.variant_rank, p.id + `); + + let filtered; + let total; + if (String(q).trim()) { + // matchesSearch also examines variant JSON-derived fields, so retain its exact + // semantics for searches while still applying safe SQL filters first. + const rows = selectRows('').all(userId ?? -1, ...filteredParams); + filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q)); + total = filtered.length; + filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit); + } else { + const groupedSql = ` + FROM playlist_items p + JOIN logical_channels l ON l.id = p.logical_channel_id + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' + AND c.category_id = p.category_id + WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} + GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`; + total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`) + .get(...filteredParams).total; + const keys = database.prepare(` + SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code, + MIN(l.display_name) AS display_name + ${groupedSql} + ORDER BY display_name, p.logical_channel_id, language_code + LIMIT ? OFFSET ? + `).all(...filteredParams, bounds.limit, bounds.offset); + if (!keys.length) { + filtered = []; + } else { + const keyFilter = `AND (${keys.map(() => + `(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)` + ).join(' OR ')})`; + const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]); + filtered = serializeChannels( + selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams), + includeHidden + ); + } } - const serialized = serializeChannels(rows, includeHidden); - const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()] - .map(language => ({ code: language.code, short: language.short, label: language.label })) - .sort((left, right) => left.label.localeCompare(right.label)); - const filtered = serialized.filter(channel => - (!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q) - ); - const persistedEpg = new Map(database.prepare(` + + const persistedEpg = filtered.length ? new Map(database.prepare(` SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data FROM live_epg_matches - `).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])); + WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')} + `).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code])) + .map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map(); for (const channel of filtered) { const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`); if (!match) continue; @@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null, language: channel.language.code }; } - const bounds = pageBounds(page, limit); - const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit); // Catalog loading must stay bounded: both enrichers perform synchronous SQLite work, // and EPG matching is especially expensive against large XMLTV catalogs. Enrich only // a genuinely targeted, bounded search; base projection data remains usable otherwise. - const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS; - const enriched = !shouldEnrich ? paged : paged.map(channel => { + const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS; + const enriched = !shouldEnrich ? filtered : filtered.map(channel => { const match = standardCatalog.findMatch(database, channel); if (match) { channel.standardName = match.name; @@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null, return { channels: enriched.map(listChannel), availableLanguages, - page: pageMetadata(filtered.length, bounds) + page: pageMetadata(total, bounds) }; } diff --git a/test/t42LiveListCpuRegression.test.js b/test/t42LiveListCpuRegression.test.js new file mode 100644 index 0000000..d45f554 --- /dev/null +++ b/test/t42LiveListCpuRegression.test.js @@ -0,0 +1,136 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { performance } = require('node:perf_hooks'); +const Database = require('better-sqlite3'); +const liveProjection = require('../server/services/liveProjectionService'); +const { + ensureColumns, + LIVE_PROJECTION_COLUMNS, + LIVE_PROJECTION_SCHEMA_SQL +} = require('../server/db/liveProjectionSchema'); + +function createProjectionDb() { + const db = new Database(':memory:'); + db.pragma('journal_mode = MEMORY'); + db.pragma('synchronous = OFF'); + db.exec(` + CREATE TABLE categories ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + category_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + parent_id TEXT, + is_hidden INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE playlist_items ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + category_id TEXT, + parent_id TEXT, + stream_icon TEXT, + stream_url TEXT, + container_extension TEXT, + rating REAL, + year TEXT, + added_at TEXT, + is_hidden INTEGER DEFAULT 0, + is_favorite INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + item_type TEXT NOT NULL + ); + `); + ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS); + db.exec(LIVE_PROJECTION_SCHEMA_SQL); + db.prepare(` + INSERT INTO categories (id, source_id, category_id, type, name) + VALUES ('cat:1', 1, 'general', 'live', 'General') + `).run(); + return db; +} + +function seedProjectedChannels(db, total) { + const logical = db.prepare(` + INSERT INTO logical_channels + (id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at) + VALUES (?, ?, ?, ?, 1, 1, 1) + `); + const item = db.prepare(` + INSERT INTO playlist_items + (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data, + logical_channel_id, grouping_fingerprint, grouping_input_hash, quality_label, quality_rank, + variant_rank, language_code, language_confidence, language_detector, language_input_hash) + VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, 'HD', 2, 0, 'en', 1, 'algo', ?) + `); + const insertAll = db.transaction(() => { + for (let index = 0; index < total; index++) { + const id = `1:${index}`; + const logicalId = `logical:${index}`; + const name = `Channel ${String(index).padStart(6, '0')}`; + logical.run(logicalId, `channel ${index}`, name, id); + item.run( + id, + String(index), + name, + `https://streams.test/${index}.m3u8`, + logicalId, + `channel ${index}`, + `group-hash-${index}`, + `language-hash-${index}` + ); + } + }); + insertAll(); +} + +test('/api/live/channels first page stays CPU-bounded for a large projected catalog', () => { + const db = createProjectionDb(); + seedProjectedChannels(db, 50000); + + try { + const cpuBefore = process.cpuUsage(); + const started = performance.now(); + const result = liveProjection.getChannelPage({ + includeHidden: false, + userId: 7, + page: 1, + limit: 100 + }, db, { + iptvOrgCatalog: { findMatch: () => null } + }); + const elapsedMs = performance.now() - started; + const cpu = process.cpuUsage(cpuBefore); + const metrics = { + rows: 50000, + returnedChannels: result.channels.length, + totalChannels: result.page.total, + elapsedMs: Math.round(elapsedMs), + cpuUserMs: Math.round(cpu.user / 1000), + cpuSystemMs: Math.round(cpu.system / 1000), + payloadBytes: Buffer.byteLength(JSON.stringify({ + filters: { availableLanguages: result.availableLanguages }, + channels: result.channels, + page: result.page + })) + }; + console.log('[T42 live list metrics]', JSON.stringify(metrics)); + + assert.equal(result.channels.length, 100); + assert.ok( + metrics.elapsedMs <= 500 && metrics.cpuUserMs <= 500, + `first live page should be bounded; measured ${JSON.stringify(metrics)}` + ); + } finally { + db.close(); + } +});