blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(live): bound live channel listing queries

Browse Source 
Optimize the live channel page path so unfiltered catalog pages are counted and paged in SQLite before serialization and enrichment.

Add the T42 CPU regression test covering a 50000-channel projected catalog.

Refs #42

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-08 17:17:16 +02:00
parent 206bbe82c2
commit 8b1daea23c
3 changed files with 219 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/db/liveProjectionSchema.js
View File

@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = {
const LIVE_PROJECTION_SCHEMA_SQL = `
CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id);
CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint);
CREATE INDEX IF NOT EXISTS idx_items_live_projection_page
ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank);
CREATE TABLE IF NOT EXISTS logical_channels (
id TEXT PRIMARY KEY,

114
server/services/liveProjectionService.js
View File

@ -506,42 +506,92 @@ function listChannel(channel) {
function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) {
database ||= defaultDatabase();
const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog;
const params = [];
let sourceFilter = '';
if (sourceId !== null) {
sourceFilter = 'AND p.source_id = ?';
params.push(sourceId);
}
const rows = database.prepare(`
const bounds = pageBounds(page, limit);
const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?';
const sourceParams = sourceId === null ? [] : [sourceId];
const visibilityFilter = includeHidden
? ''
: 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0';
const languageCodes = [...new Set(languages.filter(Boolean))];
const languageFilter = languageCodes.length
? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})`
: '';
const filteredParams = [...sourceParams, ...languageCodes];
const availableLanguageCodes = database.prepare(`
SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code
FROM playlist_items p
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter}
`).all(...sourceParams).map(row => row.language_code);
const availableLanguages = availableLanguageCodes.map(code => ({
code,
short: code === 'und' ? 'UND' : code.toUpperCase(),
label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code)
})).sort((left, right) => left.label.localeCompare(right.label));
const selectRows = extraWhere => database.prepare(`
SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
l.display_name, l.preferred_variant_id
l.display_name, l.preferred_variant_id, c.name AS category_name,
COALESCE(c.is_hidden, 0) AS category_hidden
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
AND f.item_id = p.item_id AND f.item_type = 'channel'
WHERE p.type = 'live' ${sourceFilter}
ORDER BY l.display_name, p.variant_rank
`).all(userId ?? -1, ...params);
const categoryRows = sourceId === null
? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all()
: database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId);
const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category]));
for (const row of rows) {
const category = categories.get(`${row.source_id}\0${row.category_id}`);
row.category_name = category?.name || null;
row.category_hidden = category?.is_hidden || 0;
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere}
ORDER BY l.display_name, p.logical_channel_id,
COALESCE(p.language_code, 'und'), p.variant_rank, p.id
`);
let filtered;
let total;
if (String(q).trim()) {
// matchesSearch also examines variant JSON-derived fields, so retain its exact
// semantics for searches while still applying safe SQL filters first.
const rows = selectRows('').all(userId ?? -1, ...filteredParams);
filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q));
total = filtered.length;
filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
} else {
const groupedSql = `
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter}
GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`;
total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`)
.get(...filteredParams).total;
const keys = database.prepare(`
SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code,
MIN(l.display_name) AS display_name
${groupedSql}
ORDER BY display_name, p.logical_channel_id, language_code
LIMIT ? OFFSET ?
`).all(...filteredParams, bounds.limit, bounds.offset);
if (!keys.length) {
filtered = [];
} else {
const keyFilter = `AND (${keys.map(() =>
`(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)`
).join(' OR ')})`;
const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]);
filtered = serializeChannels(
selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams),
includeHidden
);
}
}
const serialized = serializeChannels(rows, includeHidden);
const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()]
.map(language => ({ code: language.code, short: language.short, label: language.label }))
.sort((left, right) => left.label.localeCompare(right.label));
const filtered = serialized.filter(channel =>
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
);
const persistedEpg = new Map(database.prepare(`
const persistedEpg = filtered.length ? new Map(database.prepare(`
SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data
FROM live_epg_matches
`).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match]));
WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')}
`).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code]))
.map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map();
for (const channel of filtered) {
const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`);
if (!match) continue;
@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
language: channel.language.code
};
}
const bounds = pageBounds(page, limit);
const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
// Catalog loading must stay bounded: both enrichers perform synchronous SQLite work,
// and EPG matching is especially expensive against large XMLTV catalogs. Enrich only
// a genuinely targeted, bounded search; base projection data remains usable otherwise.
const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? paged : paged.map(channel => {
const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? filtered : filtered.map(channel => {
const match = standardCatalog.findMatch(database, channel);
if (match) {
channel.standardName = match.name;
@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
return {
channels: enriched.map(listChannel),
availableLanguages,
page: pageMetadata(filtered.length, bounds)
page: pageMetadata(total, bounds)
};
}