blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(live): bound live channel listing queries

Browse Source 
Optimize the live channel page path so unfiltered catalog pages are counted and paged in SQLite before serialization and enrichment.

Add the T42 CPU regression test covering a 50000-channel projected catalog.

Refs #42

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-08 17:17:16 +02:00
parent 206bbe82c2
commit 8b1daea23c
3 changed files with 219 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/db/liveProjectionSchema.js
View File

@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = {
const LIVE_PROJECTION_SCHEMA_SQL = `
CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id);
CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint);
CREATE INDEX IF NOT EXISTS idx_items_live_projection_page
ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank);
CREATE TABLE IF NOT EXISTS logical_channels (
id TEXT PRIMARY KEY,

114
server/services/liveProjectionService.js
View File

@ -506,42 +506,92 @@ function listChannel(channel) {
function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) {
database ||= defaultDatabase();
const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog;
const params = [];
let sourceFilter = '';
if (sourceId !== null) {
sourceFilter = 'AND p.source_id = ?';
params.push(sourceId);
}
const rows = database.prepare(`
const bounds = pageBounds(page, limit);
const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?';
const sourceParams = sourceId === null ? [] : [sourceId];
const visibilityFilter = includeHidden
? ''
: 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0';
const languageCodes = [...new Set(languages.filter(Boolean))];
const languageFilter = languageCodes.length
? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})`
: '';
const filteredParams = [...sourceParams, ...languageCodes];
const availableLanguageCodes = database.prepare(`
SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code
FROM playlist_items p
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter}
`).all(...sourceParams).map(row => row.language_code);
const availableLanguages = availableLanguageCodes.map(code => ({
code,
short: code === 'und' ? 'UND' : code.toUpperCase(),
label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code)
})).sort((left, right) => left.label.localeCompare(right.label));
const selectRows = extraWhere => database.prepare(`
SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
l.display_name, l.preferred_variant_id
l.display_name, l.preferred_variant_id, c.name AS category_name,
COALESCE(c.is_hidden, 0) AS category_hidden
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
AND f.item_id = p.item_id AND f.item_type = 'channel'
WHERE p.type = 'live' ${sourceFilter}
ORDER BY l.display_name, p.variant_rank
`).all(userId ?? -1, ...params);
const categoryRows = sourceId === null
? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all()
: database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId);
const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category]));
for (const row of rows) {
const category = categories.get(`${row.source_id}\0${row.category_id}`);
row.category_name = category?.name || null;
row.category_hidden = category?.is_hidden || 0;
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere}
ORDER BY l.display_name, p.logical_channel_id,
COALESCE(p.language_code, 'und'), p.variant_rank, p.id
`);
let filtered;
let total;
if (String(q).trim()) {
// matchesSearch also examines variant JSON-derived fields, so retain its exact
// semantics for searches while still applying safe SQL filters first.
const rows = selectRows('').all(userId ?? -1, ...filteredParams);
filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q));
total = filtered.length;
filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
} else {
const groupedSql = `
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter}
GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`;
total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`)
.get(...filteredParams).total;
const keys = database.prepare(`
SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code,
MIN(l.display_name) AS display_name
${groupedSql}
ORDER BY display_name, p.logical_channel_id, language_code
LIMIT ? OFFSET ?
`).all(...filteredParams, bounds.limit, bounds.offset);
if (!keys.length) {
filtered = [];
} else {
const keyFilter = `AND (${keys.map(() =>
`(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)`
).join(' OR ')})`;
const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]);
filtered = serializeChannels(
selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams),
includeHidden
);
}
}
const serialized = serializeChannels(rows, includeHidden);
const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()]
.map(language => ({ code: language.code, short: language.short, label: language.label }))
.sort((left, right) => left.label.localeCompare(right.label));
const filtered = serialized.filter(channel =>
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
);
const persistedEpg = new Map(database.prepare(`
const persistedEpg = filtered.length ? new Map(database.prepare(`
SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data
FROM live_epg_matches
`).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match]));
WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')}
`).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code]))
.map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map();
for (const channel of filtered) {
const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`);
if (!match) continue;
@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
language: channel.language.code
};
}
const bounds = pageBounds(page, limit);
const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
// Catalog loading must stay bounded: both enrichers perform synchronous SQLite work,
// and EPG matching is especially expensive against large XMLTV catalogs. Enrich only
// a genuinely targeted, bounded search; base projection data remains usable otherwise.
const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? paged : paged.map(channel => {
const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? filtered : filtered.map(channel => {
const match = standardCatalog.findMatch(database, channel);
if (match) {
channel.standardName = match.name;
@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
return {
channels: enriched.map(listChannel),
availableLanguages,
page: pageMetadata(filtered.length, bounds)
page: pageMetadata(total, bounds)
};
}

136
test/t42LiveListCpuRegression.test.js Normal file
View File

@ -0,0 +1,136 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { performance } = require('node:perf_hooks');
const Database = require('better-sqlite3');
const liveProjection = require('../server/services/liveProjectionService');
const {
ensureColumns,
LIVE_PROJECTION_COLUMNS,
LIVE_PROJECTION_SCHEMA_SQL
} = require('../server/db/liveProjectionSchema');
function createProjectionDb() {
const db = new Database(':memory:');
db.pragma('journal_mode = MEMORY');
db.pragma('synchronous = OFF');
db.exec(`
CREATE TABLE categories (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
category_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
parent_id TEXT,
is_hidden INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE playlist_items (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
category_id TEXT,
parent_id TEXT,
stream_icon TEXT,
stream_url TEXT,
container_extension TEXT,
rating REAL,
year TEXT,
added_at TEXT,
is_hidden INTEGER DEFAULT 0,
is_favorite INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE favorites (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
item_type TEXT NOT NULL
);
`);
ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
db.exec(LIVE_PROJECTION_SCHEMA_SQL);
db.prepare(`
INSERT INTO categories (id, source_id, category_id, type, name)
VALUES ('cat:1', 1, 'general', 'live', 'General')
`).run();
return db;
}
function seedProjectedChannels(db, total) {
const logical = db.prepare(`
INSERT INTO logical_channels
(id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at)
VALUES (?, ?, ?, ?, 1, 1, 1)
`);
const item = db.prepare(`
INSERT INTO playlist_items
(id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data,
logical_channel_id, grouping_fingerprint, grouping_input_hash, quality_label, quality_rank,
variant_rank, language_code, language_confidence, language_detector, language_input_hash)
VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, 'HD', 2, 0, 'en', 1, 'algo', ?)
`);
const insertAll = db.transaction(() => {
for (let index = 0; index < total; index++) {
const id = `1:${index}`;
const logicalId = `logical:${index}`;
const name = `Channel ${String(index).padStart(6, '0')}`;
logical.run(logicalId, `channel ${index}`, name, id);
item.run(
id,
String(index),
name,
`https://streams.test/${index}.m3u8`,
logicalId,
`channel ${index}`,
`group-hash-${index}`,
`language-hash-${index}`
);
}
});
insertAll();
}
test('/api/live/channels first page stays CPU-bounded for a large projected catalog', () => {
const db = createProjectionDb();
seedProjectedChannels(db, 50000);
try {
const cpuBefore = process.cpuUsage();
const started = performance.now();
const result = liveProjection.getChannelPage({
includeHidden: false,
userId: 7,
page: 1,
limit: 100
}, db, {
iptvOrgCatalog: { findMatch: () => null }
});
const elapsedMs = performance.now() - started;
const cpu = process.cpuUsage(cpuBefore);
const metrics = {
rows: 50000,
returnedChannels: result.channels.length,
totalChannels: result.page.total,
elapsedMs: Math.round(elapsedMs),
cpuUserMs: Math.round(cpu.user / 1000),
cpuSystemMs: Math.round(cpu.system / 1000),
payloadBytes: Buffer.byteLength(JSON.stringify({
filters: { availableLanguages: result.availableLanguages },
channels: result.channels,
page: result.page
}))
};
console.log('[T42 live list metrics]', JSON.stringify(metrics));
assert.equal(result.channels.length, 100);
assert.ok(
metrics.elapsedMs <= 500 && metrics.cpuUserMs <= 500,
`first live page should be bounded; measured ${JSON.stringify(metrics)}`
);
} finally {
db.close();
}
});