blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/t42-live-list-cpu-regression' into develop (#42)

Browse Source
This commit is contained in:
blomios
2026-10-08 17:17:22 +02:00
parent 206bbe82c2 8b1daea23c
commit 9e6b1d4593
3 changed files with 219 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/db/liveProjectionSchema.js
View File

@ -24,6 +24,8 @@ const CATALOG_PROJECTION_STATE_COLUMNS = {
const LIVE_PROJECTION_SCHEMA_SQL = `
CREATE INDEX IF NOT EXISTS idx_items_logical_channel ON playlist_items(logical_channel_id);
CREATE INDEX IF NOT EXISTS idx_items_grouping_fingerprint ON playlist_items(grouping_fingerprint);
CREATE INDEX IF NOT EXISTS idx_items_live_projection_page
ON playlist_items(type, logical_channel_id, language_code, source_id, variant_rank);
CREATE TABLE IF NOT EXISTS logical_channels (
id TEXT PRIMARY KEY,

114
server/services/liveProjectionService.js
View File

@ -506,42 +506,92 @@ function listChannel(channel) {
function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) {
database ||= defaultDatabase();
const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog;
const params = [];
let sourceFilter = '';
if (sourceId !== null) {
sourceFilter = 'AND p.source_id = ?';
params.push(sourceId);
}
const rows = database.prepare(`
const bounds = pageBounds(page, limit);
const sourceFilter = sourceId === null ? '' : 'AND p.source_id = ?';
const sourceParams = sourceId === null ? [] : [sourceId];
const visibilityFilter = includeHidden
? ''
: 'AND COALESCE(p.is_hidden, 0) = 0 AND COALESCE(c.is_hidden, 0) = 0';
const languageCodes = [...new Set(languages.filter(Boolean))];
const languageFilter = languageCodes.length
? `AND COALESCE(p.language_code, 'und') IN (${languageCodes.map(() => '?').join(', ')})`
: '';
const filteredParams = [...sourceParams, ...languageCodes];
const availableLanguageCodes = database.prepare(`
SELECT DISTINCT COALESCE(p.language_code, 'und') AS language_code
FROM playlist_items p
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter}
`).all(...sourceParams).map(row => row.language_code);
const availableLanguages = availableLanguageCodes.map(code => ({
code,
short: code === 'und' ? 'UND' : code.toUpperCase(),
label: code === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(code)
})).sort((left, right) => left.label.localeCompare(right.label));
const selectRows = extraWhere => database.prepare(`
SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
l.display_name, l.preferred_variant_id
l.display_name, l.preferred_variant_id, c.name AS category_name,
COALESCE(c.is_hidden, 0) AS category_hidden
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
AND f.item_id = p.item_id AND f.item_type = 'channel'
WHERE p.type = 'live' ${sourceFilter}
ORDER BY l.display_name, p.variant_rank
`).all(userId ?? -1, ...params);
const categoryRows = sourceId === null
? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all()
: database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId);
const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category]));
for (const row of rows) {
const category = categories.get(`${row.source_id}\0${row.category_id}`);
row.category_name = category?.name || null;
row.category_hidden = category?.is_hidden || 0;
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter} ${extraWhere}
ORDER BY l.display_name, p.logical_channel_id,
COALESCE(p.language_code, 'und'), p.variant_rank, p.id
`);
let filtered;
let total;
if (String(q).trim()) {
// matchesSearch also examines variant JSON-derived fields, so retain its exact
// semantics for searches while still applying safe SQL filters first.
const rows = selectRows('').all(userId ?? -1, ...filteredParams);
filtered = serializeChannels(rows, includeHidden).filter(channel => matchesSearch(channel, q));
total = filtered.length;
filtered = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
} else {
const groupedSql = `
FROM playlist_items p
JOIN logical_channels l ON l.id = p.logical_channel_id
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live'
AND c.category_id = p.category_id
WHERE p.type = 'live' ${sourceFilter} ${visibilityFilter} ${languageFilter}
GROUP BY p.logical_channel_id, COALESCE(p.language_code, 'und')`;
total = database.prepare(`SELECT COUNT(*) AS total FROM (SELECT 1 ${groupedSql})`)
.get(...filteredParams).total;
const keys = database.prepare(`
SELECT p.logical_channel_id, COALESCE(p.language_code, 'und') AS language_code,
MIN(l.display_name) AS display_name
${groupedSql}
ORDER BY display_name, p.logical_channel_id, language_code
LIMIT ? OFFSET ?
`).all(...filteredParams, bounds.limit, bounds.offset);
if (!keys.length) {
filtered = [];
} else {
const keyFilter = `AND (${keys.map(() =>
`(p.logical_channel_id = ? AND COALESCE(p.language_code, 'und') = ?)`
).join(' OR ')})`;
const keyParams = keys.flatMap(key => [key.logical_channel_id, key.language_code]);
filtered = serializeChannels(
selectRows(keyFilter).all(userId ?? -1, ...filteredParams, ...keyParams),
includeHidden
);
}
}
const serialized = serializeChannels(rows, includeHidden);
const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()]
.map(language => ({ code: language.code, short: language.short, label: language.label }))
.sort((left, right) => left.label.localeCompare(right.label));
const filtered = serialized.filter(channel =>
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
);
const persistedEpg = new Map(database.prepare(`
const persistedEpg = filtered.length ? new Map(database.prepare(`
SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data
FROM live_epg_matches
`).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match]));
WHERE ${filtered.map(() => '(logical_channel_id = ? AND language_code = ?)').join(' OR ')}
`).all(...filtered.flatMap(channel => [channel.logicalChannelId, channel.language.code]))
.map(match => [`${match.logical_channel_id}\0${match.language_code}`, match])) : new Map();
for (const channel of filtered) {
const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`);
if (!match) continue;
@ -556,13 +606,11 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
language: channel.language.code
};
}
const bounds = pageBounds(page, limit);
const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
// Catalog loading must stay bounded: both enrichers perform synchronous SQLite work,
// and EPG matching is especially expensive against large XMLTV catalogs. Enrich only
// a genuinely targeted, bounded search; base projection data remains usable otherwise.
const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? paged : paged.map(channel => {
const shouldEnrich = Boolean(String(q).trim()) && total <= MAX_SYNCHRONOUS_ENRICHMENTS;
const enriched = !shouldEnrich ? filtered : filtered.map(channel => {
const match = standardCatalog.findMatch(database, channel);
if (match) {
channel.standardName = match.name;
@ -573,7 +621,7 @@ function getChannelPage({ sourceId = null, includeHidden = false, userId = null,
return {
channels: enriched.map(listChannel),
availableLanguages,
page: pageMetadata(filtered.length, bounds)
page: pageMetadata(total, bounds)
};
}

136
test/t42LiveListCpuRegression.test.js Normal file
View File

@ -0,0 +1,136 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { performance } = require('node:perf_hooks');
const Database = require('better-sqlite3');
const liveProjection = require('../server/services/liveProjectionService');
const {
ensureColumns,
LIVE_PROJECTION_COLUMNS,
LIVE_PROJECTION_SCHEMA_SQL
} = require('../server/db/liveProjectionSchema');
function createProjectionDb() {
const db = new Database(':memory:');
db.pragma('journal_mode = MEMORY');
db.pragma('synchronous = OFF');
db.exec(`
CREATE TABLE categories (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
category_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
parent_id TEXT,
is_hidden INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE playlist_items (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
category_id TEXT,
parent_id TEXT,
stream_icon TEXT,
stream_url TEXT,
container_extension TEXT,
rating REAL,
year TEXT,
added_at TEXT,
is_hidden INTEGER DEFAULT 0,
is_favorite INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE favorites (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
item_type TEXT NOT NULL
);
`);
ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
db.exec(LIVE_PROJECTION_SCHEMA_SQL);
db.prepare(`
INSERT INTO categories (id, source_id, category_id, type, name)
VALUES ('cat:1', 1, 'general', 'live', 'General')
`).run();
return db;
}
function seedProjectedChannels(db, total) {
const logical = db.prepare(`
INSERT INTO logical_channels
(id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at)
VALUES (?, ?, ?, ?, 1, 1, 1)
`);
const item = db.prepare(`
INSERT INTO playlist_items
(id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data,
logical_channel_id, grouping_fingerprint, grouping_input_hash, quality_label, quality_rank,
variant_rank, language_code, language_confidence, language_detector, language_input_hash)
VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, 'HD', 2, 0, 'en', 1, 'algo', ?)
`);
const insertAll = db.transaction(() => {
for (let index = 0; index < total; index++) {
const id = `1:${index}`;
const logicalId = `logical:${index}`;
const name = `Channel ${String(index).padStart(6, '0')}`;
logical.run(logicalId, `channel ${index}`, name, id);
item.run(
id,
String(index),
name,
`https://streams.test/${index}.m3u8`,
logicalId,
`channel ${index}`,
`group-hash-${index}`,
`language-hash-${index}`
);
}
});
insertAll();
}
test('/api/live/channels first page stays CPU-bounded for a large projected catalog', () => {
const db = createProjectionDb();
seedProjectedChannels(db, 50000);
try {
const cpuBefore = process.cpuUsage();
const started = performance.now();
const result = liveProjection.getChannelPage({
includeHidden: false,
userId: 7,
page: 1,
limit: 100
}, db, {
iptvOrgCatalog: { findMatch: () => null }
});
const elapsedMs = performance.now() - started;
const cpu = process.cpuUsage(cpuBefore);
const metrics = {
rows: 50000,
returnedChannels: result.channels.length,
totalChannels: result.page.total,
elapsedMs: Math.round(elapsedMs),
cpuUserMs: Math.round(cpu.user / 1000),
cpuSystemMs: Math.round(cpu.system / 1000),
payloadBytes: Buffer.byteLength(JSON.stringify({
filters: { availableLanguages: result.availableLanguages },
channels: result.channels,
page: result.page
}))
};
console.log('[T42 live list metrics]', JSON.stringify(metrics));
assert.equal(result.channels.length, 100);
assert.ok(
metrics.elapsedMs <= 500 && metrics.cpuUserMs <= 500,
`first live page should be bounded; measured ${JSON.stringify(metrics)}`
);
} finally {
db.close();
}
});