fix(live): index category lookups for catalog search (#49)
Avoid repeated category scans while filtering the paginated Live catalog, and keep the search timing and query-plan reproductions in the regression suite. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
360
test/live-catalog-search-performance.test.js
Normal file
360
test/live-catalog-search-performance.test.js
Normal file
@ -0,0 +1,360 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { performance } = require('node:perf_hooks');
|
||||
const { spawn } = require('node:child_process');
|
||||
const Database = require('better-sqlite3');
|
||||
const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const SCALE = process.env.NODECAST_LIVE_SEARCH_SIZES
|
||||
? process.env.NODECAST_LIVE_SEARCH_SIZES.split(',').map(Number) : [1200, 10000, 50000, 100000];
|
||||
const TERMS = ['news', 'sports', 'local', 'kids', 'world'];
|
||||
const SEARCH_LIMIT_MS = 10000;
|
||||
const BROWSER_TERMS = ['news', 'sports', 'local', 'kids', 'world'];
|
||||
|
||||
async function launchServer(t, dir) {
|
||||
const child = spawn(process.execPath, ['server/index.js'], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, PORT: '0', NODECAST_DATA_DIR: dir,
|
||||
NODECAST_DISABLE_BACKGROUND_JOBS: 'true', JWT_SECRET: '', SESSION_SECRET: '',
|
||||
OIDC_ISSUER_URL: '', AUTH_COOKIE_SECURE: 'false' },
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
});
|
||||
t.after(async () => {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
const closed = new Promise(resolve => child.once('close', resolve));
|
||||
child.kill();
|
||||
await closed;
|
||||
}
|
||||
});
|
||||
let output = '';
|
||||
const startedAt = performance.now();
|
||||
const base = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`Startup timed out: ${output}`)), 30000);
|
||||
child.stdout.on('data', chunk => {
|
||||
output += chunk;
|
||||
const match = output.match(/server running on http:\/\/localhost:(\d+)/i);
|
||||
if (match) { clearTimeout(timer); resolve(`http://127.0.0.1:${match[1]}`); }
|
||||
});
|
||||
child.stderr.on('data', chunk => { output += chunk; });
|
||||
child.once('exit', code => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(`Server exited ${code}: ${output}`));
|
||||
});
|
||||
});
|
||||
return { child, base, processReadyMs: performance.now() - startedAt };
|
||||
}
|
||||
|
||||
async function stopServer(child) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
const closed = new Promise(resolve => child.once('close', resolve));
|
||||
child.kill('SIGTERM');
|
||||
await closed;
|
||||
}
|
||||
|
||||
async function apiRequest(base, cookie, query = {}) {
|
||||
const params = new URLSearchParams(query);
|
||||
const start = performance.now();
|
||||
const response = await fetch(`${base}/api/channels/live-catalog?${params}`, {
|
||||
headers: { Cookie: cookie }
|
||||
});
|
||||
const text = await response.text();
|
||||
return { status: response.status, totalMs: performance.now() - start,
|
||||
bytes: Buffer.byteLength(text), body: JSON.parse(text) };
|
||||
}
|
||||
|
||||
function seedDelta(dir, from, to) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
const insertCategory = db.prepare(`
|
||||
INSERT OR IGNORE INTO categories (id, source_id, category_id, type, name, is_hidden)
|
||||
VALUES (?, 1, ?, 'live', ?, 0)
|
||||
`);
|
||||
const insertItem = db.prepare(`
|
||||
INSERT INTO playlist_items (id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data)
|
||||
VALUES (?, 1, ?, 'live', ?, ?, ?, 0, '{}')
|
||||
`);
|
||||
const themes = ['News', 'Sports', 'Local', 'Kids', 'World'];
|
||||
const groupCount = 20;
|
||||
const insert = db.transaction(() => {
|
||||
for (let group = 0; group < groupCount; group++) {
|
||||
const categoryId = `bouquet-${String(group).padStart(2, '0')}`;
|
||||
insertCategory.run(`1:${categoryId}`, categoryId, `Bouquet ${String(group + 1).padStart(2, '0')}`);
|
||||
}
|
||||
for (let index = from; index < to; index++) {
|
||||
const id = `live-${String(index).padStart(6, '0')}`;
|
||||
const theme = themes[index % themes.length];
|
||||
const categoryId = `bouquet-${String(index % groupCount).padStart(2, '0')}`;
|
||||
const region = `Region ${String(index % 12 + 1).padStart(2, '0')}`;
|
||||
insertItem.run(`1:${id}`, id, `${theme} ${region} Channel ${String(index).padStart(6, '0')} HD`,
|
||||
categoryId, `http://streams.test/${id}`);
|
||||
}
|
||||
});
|
||||
insert();
|
||||
db.close();
|
||||
}
|
||||
|
||||
function removeSearchIndexes(dir) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
db.exec(`DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ai;
|
||||
DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ad;
|
||||
DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_au;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_ai;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_ad;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_au;
|
||||
DROP TABLE IF EXISTS live_catalog_playlist_items_fts;
|
||||
DROP TABLE IF EXISTS live_catalog_categories_fts;`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
function searchIndexSnapshot(dir) {
|
||||
const db = new Database(path.join(dir, 'content.db'), { readonly: true });
|
||||
const schemaVersion = db.pragma('schema_version', { simple: true });
|
||||
const definitions = db.prepare(`SELECT name, sql FROM sqlite_master WHERE name LIKE 'live_catalog_%'
|
||||
ORDER BY name`).all();
|
||||
const playlistRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n;
|
||||
const categoryRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts').get().n;
|
||||
db.close();
|
||||
return { schemaVersion, definitions, playlistRows, categoryRows };
|
||||
}
|
||||
|
||||
function endpointKind(url) {
|
||||
if (url.pathname === '/api/channels/live-catalog') return 'live-catalog';
|
||||
if (url.pathname === '/api/channels/hidden') return 'channels/hidden';
|
||||
if (url.pathname === '/api/favorites') return 'favorites';
|
||||
if (url.pathname.startsWith('/api/proxy/epg/')) return 'epg';
|
||||
if (url.pathname === '/api/history') return 'history';
|
||||
return null;
|
||||
}
|
||||
|
||||
async function runBrowserJourney(chromium, base, dir, size) {
|
||||
const context = await chromium.newContext();
|
||||
const page = await context.newPage();
|
||||
const requests = [];
|
||||
const byRequest = new Map();
|
||||
const responseTasks = [];
|
||||
page.on('request', request => {
|
||||
const url = new URL(request.url());
|
||||
const kind = endpointKind(url);
|
||||
if (!kind) return;
|
||||
const entry = { kind, method: request.method(), path: url.pathname,
|
||||
query: Object.fromEntries(url.searchParams), startedAt: performance.now() };
|
||||
byRequest.set(request, entry);
|
||||
requests.push(entry);
|
||||
});
|
||||
page.on('response', response => {
|
||||
const entry = byRequest.get(response.request());
|
||||
if (!entry) return;
|
||||
entry.status = response.status();
|
||||
entry.durationMs = Number((performance.now() - entry.startedAt).toFixed(2));
|
||||
const task = response.body().then(body => { entry.responseBytes = body.length; }, () => { entry.responseBytes = null; });
|
||||
responseTasks.push(task);
|
||||
});
|
||||
await page.route('**/api/proxy/epg/**', route => route.fulfill({
|
||||
status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ channels: [], programmes: [] })
|
||||
}));
|
||||
await page.route('https://cdn.jsdelivr.net/npm/hls.js@1.5.7/dist/hls.min.js', route => route.abort());
|
||||
await page.route('https://fonts.**', route => route.abort());
|
||||
|
||||
const blankContext = { cookies: (await context.cookies()).length,
|
||||
localStorage: await page.evaluate(() => Object.keys(localStorage).length).catch(() => 0) };
|
||||
const homeStarted = performance.now();
|
||||
await page.goto(`${base}/login.html`);
|
||||
await page.locator('#username').fill('SearchPerf');
|
||||
await page.locator('#password').fill('password123');
|
||||
await page.locator('#password').press('Enter');
|
||||
await page.waitForURL(url => url.pathname === '/');
|
||||
await page.waitForFunction(() => window.app?.currentPage === 'home'
|
||||
&& document.querySelector('#favorite-channels-list')
|
||||
&& !document.querySelector('#favorite-channels-list .loading-state'));
|
||||
await page.waitForFunction(() => window.app?.epgGuide?.lastRefreshTime ||
|
||||
document.querySelector('#epg-container')?.textContent?.includes('No EPG'));
|
||||
const homeReadyMs = performance.now() - homeStarted;
|
||||
const homeNetwork = requests.slice();
|
||||
|
||||
const liveStarted = performance.now();
|
||||
const liveRequestIndex = requests.length;
|
||||
await page.locator('.nav-link[data-page="live"]').click();
|
||||
await page.waitForFunction(() => window.app?.currentPage === 'live'
|
||||
&& window.app.channelList.channels.length === 200
|
||||
&& !window.app.channelList.isLoading
|
||||
&& document.querySelector('#channel-list')?.getAttribute('aria-busy') === 'false');
|
||||
const liveReadyMs = performance.now() - liveStarted;
|
||||
const initialLiveNetwork = requests.slice(liveRequestIndex);
|
||||
|
||||
const beforeIndexes = searchIndexSnapshot(dir);
|
||||
const searches = [];
|
||||
for (const term of BROWSER_TERMS) {
|
||||
const firstRequest = requests.length;
|
||||
const startedAt = performance.now();
|
||||
await page.locator('#channel-search').fill(term);
|
||||
try { await page.waitForFunction(value => {
|
||||
const list = window.app?.channelList;
|
||||
const container = document.querySelector('#channel-list');
|
||||
return list?.catalogQuery === value && !list.isLoading && list.channels.length === 200
|
||||
&& container?.getAttribute('aria-busy') === 'false';
|
||||
}, term, { timeout: 5000 }); } catch (error) {
|
||||
const browserState = await page.evaluate(() => ({ currentPage: window.app?.currentPage,
|
||||
searchValue: document.querySelector('#channel-search')?.value,
|
||||
catalogQuery: window.app?.channelList?.catalogQuery,
|
||||
channelCount: window.app?.channelList?.channels?.length,
|
||||
isLoading: window.app?.channelList?.isLoading,
|
||||
status: document.querySelector('#channel-list')?.getAttribute('aria-busy'),
|
||||
groups: window.app?.channelList?.sortedGroups?.length }));
|
||||
console.log(`[live-search-browser-debug] ${JSON.stringify({ size, term, browserState,
|
||||
requests: requests.slice(firstRequest).map(entry => ({ kind: entry.kind, query: entry.query, status: entry.status })) })}`);
|
||||
throw error;
|
||||
}
|
||||
await page.waitForTimeout(75);
|
||||
const settledAt = performance.now();
|
||||
const current = requests.slice(firstRequest);
|
||||
await Promise.all(responseTasks);
|
||||
const indexState = searchIndexSnapshot(dir);
|
||||
searches.push({ term, inputToStableDomMs: Number((settledAt - startedAt).toFixed(2)),
|
||||
channels: await page.evaluate(() => window.app.channelList.channels.length),
|
||||
domChannelGroups: await page.locator('#channel-list .channel-group').count(),
|
||||
network: current.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
ftsSchemaVersion: indexState.schemaVersion,
|
||||
ftsRows: { playlist: indexState.playlistRows, categories: indexState.categoryRows },
|
||||
schemaUnchanged: indexState.schemaVersion === beforeIndexes.schemaVersion
|
||||
&& JSON.stringify(indexState.definitions) === JSON.stringify(beforeIndexes.definitions)
|
||||
&& indexState.playlistRows === beforeIndexes.playlistRows
|
||||
&& indexState.categoryRows === beforeIndexes.categoryRows });
|
||||
}
|
||||
await Promise.all(responseTasks);
|
||||
const afterIndexes = searchIndexSnapshot(dir);
|
||||
const report = { rows: size, blankContext: blankContext.cookies === 0 && blankContext.localStorage === 0,
|
||||
homeReadyMs: Number(homeReadyMs.toFixed(2)), liveReadyMs: Number(liveReadyMs.toFixed(2)),
|
||||
homeNetwork: homeNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
initialLiveNetwork: initialLiveNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
searches, ftsUnchangedAcrossSearches: JSON.stringify(beforeIndexes) === JSON.stringify(afterIndexes) };
|
||||
console.log(`[live-search-browser] ${JSON.stringify(report)}`);
|
||||
assert.equal(report.blankContext, true, 'the browser must start without cookies or localStorage');
|
||||
assert.ok(searches.every(search => search.schemaUnchanged), 'search requests must not recreate or backfill FTS indexes');
|
||||
assert.equal(report.ftsUnchangedAcrossSearches, true);
|
||||
await context.close();
|
||||
return report;
|
||||
}
|
||||
|
||||
function measureService(dir, userId, term) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
ensureSearch(db);
|
||||
let sqlMs = 0;
|
||||
const prepare = db.prepare.bind(db);
|
||||
db.prepare = sql => {
|
||||
const statement = prepare(sql);
|
||||
if (!sql.includes('WITH live_sources AS')) return statement;
|
||||
return { all: (...args) => {
|
||||
const startedAt = performance.now();
|
||||
try { return statement.all(...args); }
|
||||
finally { sqlMs += performance.now() - startedAt; }
|
||||
} };
|
||||
};
|
||||
const allSources = [{ id: 1, type: 'm3u', enabled: true }];
|
||||
const options = parseQuery({ q: term, limit: '100' }, userId);
|
||||
const startedAt = performance.now();
|
||||
const result = getLiveCatalog(db, allSources, options);
|
||||
const serviceMs = performance.now() - startedAt;
|
||||
db.close();
|
||||
return { sqlMs, serviceMs, returned: result.items.length, hasMore: result.page.hasMore };
|
||||
}
|
||||
|
||||
test('live catalog repeated searches record first-use FTS backfill and per-query timing at 1.2k–100k rows', {
|
||||
skip: process.env.NODECAST_LIVE_SEARCH_DIAGNOSTIC === '1' ? false : 'Opt-in diagnostic benchmark',
|
||||
timeout: 300000
|
||||
}, async t => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-perf-'));
|
||||
fs.writeFileSync(path.join(dir, 'db.json'), JSON.stringify({
|
||||
sources: [
|
||||
{ id: 1, type: 'm3u', name: 'Fixture', url: 'http://127.0.0.1:1/unused.m3u', enabled: true },
|
||||
{ id: 2, type: 'epg', name: 'Fixture EPG', url: 'http://epg.fixture.invalid/guide.xml', enabled: true }
|
||||
],
|
||||
hiddenItems: [], favorites: [], users: [], nextId: 3
|
||||
}));
|
||||
let activeChild;
|
||||
t.after(async () => {
|
||||
if (activeChild) await stopServer(activeChild);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
let app = await launchServer(t, dir);
|
||||
activeChild = app.child;
|
||||
const setup = await fetch(`${app.base}/api/auth/setup`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ username: 'SearchPerf', password: 'password123' })
|
||||
});
|
||||
assert.equal(setup.status, 201);
|
||||
const setupBody = await setup.json();
|
||||
const cookie = setup.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0];
|
||||
const initialized = await apiRequest(app.base, cookie);
|
||||
assert.equal(initialized.status, 200);
|
||||
|
||||
const reports = [];
|
||||
let previousSize = 0;
|
||||
for (const size of SCALE) {
|
||||
seedDelta(dir, previousSize, size);
|
||||
previousSize = size;
|
||||
for (const [round, term] of [...TERMS, ...TERMS].entries()) {
|
||||
const response = await apiRequest(app.base, cookie, { q: term });
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.items.length, 100);
|
||||
assert.equal(response.body.page.hasMore, true);
|
||||
const sql = measureService(dir, setupBody.user.id, term);
|
||||
const expectedMatches = Math.floor((size - 1 - TERMS.indexOf(term)) / TERMS.length) + 1;
|
||||
const report = { rows: size, phase: round < TERMS.length ? 'first distinct pass' : 'repeat pass',
|
||||
term, expectedMatches, returned: response.body.items.length, hasMore: response.body.page.hasMore,
|
||||
bytes: response.bytes, httpMs: Number(response.totalMs.toFixed(2)),
|
||||
sqlMs: Number(sql.sqlMs.toFixed(2)), serviceMs: Number(sql.serviceMs.toFixed(2)) };
|
||||
reports.push(report);
|
||||
console.log(`[live-search-diagnostic] ${JSON.stringify(report)}`);
|
||||
}
|
||||
if (process.env.NODECAST_LIVE_SEARCH_BROWSER === '1' && (size === 1200 || size === 100000)) {
|
||||
const chromium = require('playwright').chromium;
|
||||
const browser = await chromium.launch({ headless: true,
|
||||
executablePath: process.env.NODECAST_BROWSER_EXECUTABLE || undefined });
|
||||
try {
|
||||
const browserReport = await runBrowserJourney(browser, app.base, dir, size);
|
||||
assert.ok(browserReport.searches.length === BROWSER_TERMS.length);
|
||||
} finally { await browser.close(); }
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate a pre-FTS database on process restart. sqlite.js creates and backfills
|
||||
// both trigram indexes on the first catalog request after the server is ready.
|
||||
await stopServer(activeChild);
|
||||
activeChild = null;
|
||||
removeSearchIndexes(dir);
|
||||
app = await launchServer(t, dir);
|
||||
activeChild = app.child;
|
||||
const login = await fetch(`${app.base}/api/auth/login`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ username: 'SearchPerf', password: 'password123' })
|
||||
});
|
||||
assert.equal(login.status, 200);
|
||||
const loginCookie = login.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0];
|
||||
const backfillStarted = performance.now();
|
||||
const firstAfterRestart = await apiRequest(app.base, loginCookie);
|
||||
const backfillMs = performance.now() - backfillStarted;
|
||||
assert.equal(firstAfterRestart.status, 200);
|
||||
const indexDb = new Database(path.join(dir, 'content.db'), { readonly: true });
|
||||
const indexes = indexDb.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'live_catalog_%_fts'`).all();
|
||||
const indexedPlaylistRows = indexDb.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n;
|
||||
indexDb.close();
|
||||
assert.equal(indexes.length, 2);
|
||||
assert.equal(indexedPlaylistRows, previousSize);
|
||||
console.log(`[live-search-diagnostic] ${JSON.stringify({ rows: previousSize, phase: 'first catalog request after dropping FTS/restart',
|
||||
httpMs: Number(firstAfterRestart.totalMs.toFixed(2)), backfillRequestMs: Number(backfillMs.toFixed(2)),
|
||||
bytes: firstAfterRestart.bytes, indexes: indexes.length, indexedPlaylistRows })}`);
|
||||
|
||||
const slow = reports.filter(report => report.httpMs >= SEARCH_LIMIT_MS);
|
||||
assert.deepEqual(slow, [], `search requests reached the observed 10s failure window: ${JSON.stringify(slow)}`);
|
||||
});
|
||||
Reference in New Issue
Block a user