fix(live): index category lookups for catalog search (#49)
Avoid repeated category scans while filtering the paginated Live catalog, and keep the search timing and query-plan reproductions in the regression suite. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@ -77,6 +77,18 @@ async function requestAllPages(base, cookie, query = {}) {
|
||||
return items;
|
||||
}
|
||||
|
||||
test('application startup creates the composite category lookup index', async t => {
|
||||
const app = await startApp(t);
|
||||
const response = await request(app.base, app.cookie);
|
||||
assert.equal(response.status, 200);
|
||||
const db = new Database(path.join(app.dir, 'content.db'), { fileMustExist: true });
|
||||
t.after(() => db.close());
|
||||
const index = db.prepare(`SELECT sql FROM sqlite_master
|
||||
WHERE type='index' AND name='idx_categories_source_type_category'`).get();
|
||||
assert.ok(index, 'startup schema should create idx_categories_source_type_category');
|
||||
assert.match(index.sql, /\(source_id,\s*type,\s*category_id\)/i);
|
||||
});
|
||||
|
||||
function seedCatalog(dir, userId) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
const insertCategory = db.prepare(`
|
||||
|
||||
360
test/live-catalog-search-performance.test.js
Normal file
360
test/live-catalog-search-performance.test.js
Normal file
@ -0,0 +1,360 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { performance } = require('node:perf_hooks');
|
||||
const { spawn } = require('node:child_process');
|
||||
const Database = require('better-sqlite3');
|
||||
const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const SCALE = process.env.NODECAST_LIVE_SEARCH_SIZES
|
||||
? process.env.NODECAST_LIVE_SEARCH_SIZES.split(',').map(Number) : [1200, 10000, 50000, 100000];
|
||||
const TERMS = ['news', 'sports', 'local', 'kids', 'world'];
|
||||
const SEARCH_LIMIT_MS = 10000;
|
||||
const BROWSER_TERMS = ['news', 'sports', 'local', 'kids', 'world'];
|
||||
|
||||
async function launchServer(t, dir) {
|
||||
const child = spawn(process.execPath, ['server/index.js'], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, PORT: '0', NODECAST_DATA_DIR: dir,
|
||||
NODECAST_DISABLE_BACKGROUND_JOBS: 'true', JWT_SECRET: '', SESSION_SECRET: '',
|
||||
OIDC_ISSUER_URL: '', AUTH_COOKIE_SECURE: 'false' },
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
});
|
||||
t.after(async () => {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
const closed = new Promise(resolve => child.once('close', resolve));
|
||||
child.kill();
|
||||
await closed;
|
||||
}
|
||||
});
|
||||
let output = '';
|
||||
const startedAt = performance.now();
|
||||
const base = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`Startup timed out: ${output}`)), 30000);
|
||||
child.stdout.on('data', chunk => {
|
||||
output += chunk;
|
||||
const match = output.match(/server running on http:\/\/localhost:(\d+)/i);
|
||||
if (match) { clearTimeout(timer); resolve(`http://127.0.0.1:${match[1]}`); }
|
||||
});
|
||||
child.stderr.on('data', chunk => { output += chunk; });
|
||||
child.once('exit', code => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(`Server exited ${code}: ${output}`));
|
||||
});
|
||||
});
|
||||
return { child, base, processReadyMs: performance.now() - startedAt };
|
||||
}
|
||||
|
||||
async function stopServer(child) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
const closed = new Promise(resolve => child.once('close', resolve));
|
||||
child.kill('SIGTERM');
|
||||
await closed;
|
||||
}
|
||||
|
||||
async function apiRequest(base, cookie, query = {}) {
|
||||
const params = new URLSearchParams(query);
|
||||
const start = performance.now();
|
||||
const response = await fetch(`${base}/api/channels/live-catalog?${params}`, {
|
||||
headers: { Cookie: cookie }
|
||||
});
|
||||
const text = await response.text();
|
||||
return { status: response.status, totalMs: performance.now() - start,
|
||||
bytes: Buffer.byteLength(text), body: JSON.parse(text) };
|
||||
}
|
||||
|
||||
function seedDelta(dir, from, to) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
const insertCategory = db.prepare(`
|
||||
INSERT OR IGNORE INTO categories (id, source_id, category_id, type, name, is_hidden)
|
||||
VALUES (?, 1, ?, 'live', ?, 0)
|
||||
`);
|
||||
const insertItem = db.prepare(`
|
||||
INSERT INTO playlist_items (id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data)
|
||||
VALUES (?, 1, ?, 'live', ?, ?, ?, 0, '{}')
|
||||
`);
|
||||
const themes = ['News', 'Sports', 'Local', 'Kids', 'World'];
|
||||
const groupCount = 20;
|
||||
const insert = db.transaction(() => {
|
||||
for (let group = 0; group < groupCount; group++) {
|
||||
const categoryId = `bouquet-${String(group).padStart(2, '0')}`;
|
||||
insertCategory.run(`1:${categoryId}`, categoryId, `Bouquet ${String(group + 1).padStart(2, '0')}`);
|
||||
}
|
||||
for (let index = from; index < to; index++) {
|
||||
const id = `live-${String(index).padStart(6, '0')}`;
|
||||
const theme = themes[index % themes.length];
|
||||
const categoryId = `bouquet-${String(index % groupCount).padStart(2, '0')}`;
|
||||
const region = `Region ${String(index % 12 + 1).padStart(2, '0')}`;
|
||||
insertItem.run(`1:${id}`, id, `${theme} ${region} Channel ${String(index).padStart(6, '0')} HD`,
|
||||
categoryId, `http://streams.test/${id}`);
|
||||
}
|
||||
});
|
||||
insert();
|
||||
db.close();
|
||||
}
|
||||
|
||||
function removeSearchIndexes(dir) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
db.exec(`DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ai;
|
||||
DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ad;
|
||||
DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_au;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_ai;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_ad;
|
||||
DROP TRIGGER IF EXISTS live_catalog_categories_fts_au;
|
||||
DROP TABLE IF EXISTS live_catalog_playlist_items_fts;
|
||||
DROP TABLE IF EXISTS live_catalog_categories_fts;`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
function searchIndexSnapshot(dir) {
|
||||
const db = new Database(path.join(dir, 'content.db'), { readonly: true });
|
||||
const schemaVersion = db.pragma('schema_version', { simple: true });
|
||||
const definitions = db.prepare(`SELECT name, sql FROM sqlite_master WHERE name LIKE 'live_catalog_%'
|
||||
ORDER BY name`).all();
|
||||
const playlistRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n;
|
||||
const categoryRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts').get().n;
|
||||
db.close();
|
||||
return { schemaVersion, definitions, playlistRows, categoryRows };
|
||||
}
|
||||
|
||||
function endpointKind(url) {
|
||||
if (url.pathname === '/api/channels/live-catalog') return 'live-catalog';
|
||||
if (url.pathname === '/api/channels/hidden') return 'channels/hidden';
|
||||
if (url.pathname === '/api/favorites') return 'favorites';
|
||||
if (url.pathname.startsWith('/api/proxy/epg/')) return 'epg';
|
||||
if (url.pathname === '/api/history') return 'history';
|
||||
return null;
|
||||
}
|
||||
|
||||
async function runBrowserJourney(chromium, base, dir, size) {
|
||||
const context = await chromium.newContext();
|
||||
const page = await context.newPage();
|
||||
const requests = [];
|
||||
const byRequest = new Map();
|
||||
const responseTasks = [];
|
||||
page.on('request', request => {
|
||||
const url = new URL(request.url());
|
||||
const kind = endpointKind(url);
|
||||
if (!kind) return;
|
||||
const entry = { kind, method: request.method(), path: url.pathname,
|
||||
query: Object.fromEntries(url.searchParams), startedAt: performance.now() };
|
||||
byRequest.set(request, entry);
|
||||
requests.push(entry);
|
||||
});
|
||||
page.on('response', response => {
|
||||
const entry = byRequest.get(response.request());
|
||||
if (!entry) return;
|
||||
entry.status = response.status();
|
||||
entry.durationMs = Number((performance.now() - entry.startedAt).toFixed(2));
|
||||
const task = response.body().then(body => { entry.responseBytes = body.length; }, () => { entry.responseBytes = null; });
|
||||
responseTasks.push(task);
|
||||
});
|
||||
await page.route('**/api/proxy/epg/**', route => route.fulfill({
|
||||
status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ channels: [], programmes: [] })
|
||||
}));
|
||||
await page.route('https://cdn.jsdelivr.net/npm/hls.js@1.5.7/dist/hls.min.js', route => route.abort());
|
||||
await page.route('https://fonts.**', route => route.abort());
|
||||
|
||||
const blankContext = { cookies: (await context.cookies()).length,
|
||||
localStorage: await page.evaluate(() => Object.keys(localStorage).length).catch(() => 0) };
|
||||
const homeStarted = performance.now();
|
||||
await page.goto(`${base}/login.html`);
|
||||
await page.locator('#username').fill('SearchPerf');
|
||||
await page.locator('#password').fill('password123');
|
||||
await page.locator('#password').press('Enter');
|
||||
await page.waitForURL(url => url.pathname === '/');
|
||||
await page.waitForFunction(() => window.app?.currentPage === 'home'
|
||||
&& document.querySelector('#favorite-channels-list')
|
||||
&& !document.querySelector('#favorite-channels-list .loading-state'));
|
||||
await page.waitForFunction(() => window.app?.epgGuide?.lastRefreshTime ||
|
||||
document.querySelector('#epg-container')?.textContent?.includes('No EPG'));
|
||||
const homeReadyMs = performance.now() - homeStarted;
|
||||
const homeNetwork = requests.slice();
|
||||
|
||||
const liveStarted = performance.now();
|
||||
const liveRequestIndex = requests.length;
|
||||
await page.locator('.nav-link[data-page="live"]').click();
|
||||
await page.waitForFunction(() => window.app?.currentPage === 'live'
|
||||
&& window.app.channelList.channels.length === 200
|
||||
&& !window.app.channelList.isLoading
|
||||
&& document.querySelector('#channel-list')?.getAttribute('aria-busy') === 'false');
|
||||
const liveReadyMs = performance.now() - liveStarted;
|
||||
const initialLiveNetwork = requests.slice(liveRequestIndex);
|
||||
|
||||
const beforeIndexes = searchIndexSnapshot(dir);
|
||||
const searches = [];
|
||||
for (const term of BROWSER_TERMS) {
|
||||
const firstRequest = requests.length;
|
||||
const startedAt = performance.now();
|
||||
await page.locator('#channel-search').fill(term);
|
||||
try { await page.waitForFunction(value => {
|
||||
const list = window.app?.channelList;
|
||||
const container = document.querySelector('#channel-list');
|
||||
return list?.catalogQuery === value && !list.isLoading && list.channels.length === 200
|
||||
&& container?.getAttribute('aria-busy') === 'false';
|
||||
}, term, { timeout: 5000 }); } catch (error) {
|
||||
const browserState = await page.evaluate(() => ({ currentPage: window.app?.currentPage,
|
||||
searchValue: document.querySelector('#channel-search')?.value,
|
||||
catalogQuery: window.app?.channelList?.catalogQuery,
|
||||
channelCount: window.app?.channelList?.channels?.length,
|
||||
isLoading: window.app?.channelList?.isLoading,
|
||||
status: document.querySelector('#channel-list')?.getAttribute('aria-busy'),
|
||||
groups: window.app?.channelList?.sortedGroups?.length }));
|
||||
console.log(`[live-search-browser-debug] ${JSON.stringify({ size, term, browserState,
|
||||
requests: requests.slice(firstRequest).map(entry => ({ kind: entry.kind, query: entry.query, status: entry.status })) })}`);
|
||||
throw error;
|
||||
}
|
||||
await page.waitForTimeout(75);
|
||||
const settledAt = performance.now();
|
||||
const current = requests.slice(firstRequest);
|
||||
await Promise.all(responseTasks);
|
||||
const indexState = searchIndexSnapshot(dir);
|
||||
searches.push({ term, inputToStableDomMs: Number((settledAt - startedAt).toFixed(2)),
|
||||
channels: await page.evaluate(() => window.app.channelList.channels.length),
|
||||
domChannelGroups: await page.locator('#channel-list .channel-group').count(),
|
||||
network: current.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
ftsSchemaVersion: indexState.schemaVersion,
|
||||
ftsRows: { playlist: indexState.playlistRows, categories: indexState.categoryRows },
|
||||
schemaUnchanged: indexState.schemaVersion === beforeIndexes.schemaVersion
|
||||
&& JSON.stringify(indexState.definitions) === JSON.stringify(beforeIndexes.definitions)
|
||||
&& indexState.playlistRows === beforeIndexes.playlistRows
|
||||
&& indexState.categoryRows === beforeIndexes.categoryRows });
|
||||
}
|
||||
await Promise.all(responseTasks);
|
||||
const afterIndexes = searchIndexSnapshot(dir);
|
||||
const report = { rows: size, blankContext: blankContext.cookies === 0 && blankContext.localStorage === 0,
|
||||
homeReadyMs: Number(homeReadyMs.toFixed(2)), liveReadyMs: Number(liveReadyMs.toFixed(2)),
|
||||
homeNetwork: homeNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
initialLiveNetwork: initialLiveNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method,
|
||||
query: entry.query, status: entry.status, responseBytes: entry.responseBytes,
|
||||
durationMs: entry.durationMs })),
|
||||
searches, ftsUnchangedAcrossSearches: JSON.stringify(beforeIndexes) === JSON.stringify(afterIndexes) };
|
||||
console.log(`[live-search-browser] ${JSON.stringify(report)}`);
|
||||
assert.equal(report.blankContext, true, 'the browser must start without cookies or localStorage');
|
||||
assert.ok(searches.every(search => search.schemaUnchanged), 'search requests must not recreate or backfill FTS indexes');
|
||||
assert.equal(report.ftsUnchangedAcrossSearches, true);
|
||||
await context.close();
|
||||
return report;
|
||||
}
|
||||
|
||||
function measureService(dir, userId, term) {
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
ensureSearch(db);
|
||||
let sqlMs = 0;
|
||||
const prepare = db.prepare.bind(db);
|
||||
db.prepare = sql => {
|
||||
const statement = prepare(sql);
|
||||
if (!sql.includes('WITH live_sources AS')) return statement;
|
||||
return { all: (...args) => {
|
||||
const startedAt = performance.now();
|
||||
try { return statement.all(...args); }
|
||||
finally { sqlMs += performance.now() - startedAt; }
|
||||
} };
|
||||
};
|
||||
const allSources = [{ id: 1, type: 'm3u', enabled: true }];
|
||||
const options = parseQuery({ q: term, limit: '100' }, userId);
|
||||
const startedAt = performance.now();
|
||||
const result = getLiveCatalog(db, allSources, options);
|
||||
const serviceMs = performance.now() - startedAt;
|
||||
db.close();
|
||||
return { sqlMs, serviceMs, returned: result.items.length, hasMore: result.page.hasMore };
|
||||
}
|
||||
|
||||
test('live catalog repeated searches record first-use FTS backfill and per-query timing at 1.2k–100k rows', {
|
||||
skip: process.env.NODECAST_LIVE_SEARCH_DIAGNOSTIC === '1' ? false : 'Opt-in diagnostic benchmark',
|
||||
timeout: 300000
|
||||
}, async t => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-perf-'));
|
||||
fs.writeFileSync(path.join(dir, 'db.json'), JSON.stringify({
|
||||
sources: [
|
||||
{ id: 1, type: 'm3u', name: 'Fixture', url: 'http://127.0.0.1:1/unused.m3u', enabled: true },
|
||||
{ id: 2, type: 'epg', name: 'Fixture EPG', url: 'http://epg.fixture.invalid/guide.xml', enabled: true }
|
||||
],
|
||||
hiddenItems: [], favorites: [], users: [], nextId: 3
|
||||
}));
|
||||
let activeChild;
|
||||
t.after(async () => {
|
||||
if (activeChild) await stopServer(activeChild);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
let app = await launchServer(t, dir);
|
||||
activeChild = app.child;
|
||||
const setup = await fetch(`${app.base}/api/auth/setup`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ username: 'SearchPerf', password: 'password123' })
|
||||
});
|
||||
assert.equal(setup.status, 201);
|
||||
const setupBody = await setup.json();
|
||||
const cookie = setup.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0];
|
||||
const initialized = await apiRequest(app.base, cookie);
|
||||
assert.equal(initialized.status, 200);
|
||||
|
||||
const reports = [];
|
||||
let previousSize = 0;
|
||||
for (const size of SCALE) {
|
||||
seedDelta(dir, previousSize, size);
|
||||
previousSize = size;
|
||||
for (const [round, term] of [...TERMS, ...TERMS].entries()) {
|
||||
const response = await apiRequest(app.base, cookie, { q: term });
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.items.length, 100);
|
||||
assert.equal(response.body.page.hasMore, true);
|
||||
const sql = measureService(dir, setupBody.user.id, term);
|
||||
const expectedMatches = Math.floor((size - 1 - TERMS.indexOf(term)) / TERMS.length) + 1;
|
||||
const report = { rows: size, phase: round < TERMS.length ? 'first distinct pass' : 'repeat pass',
|
||||
term, expectedMatches, returned: response.body.items.length, hasMore: response.body.page.hasMore,
|
||||
bytes: response.bytes, httpMs: Number(response.totalMs.toFixed(2)),
|
||||
sqlMs: Number(sql.sqlMs.toFixed(2)), serviceMs: Number(sql.serviceMs.toFixed(2)) };
|
||||
reports.push(report);
|
||||
console.log(`[live-search-diagnostic] ${JSON.stringify(report)}`);
|
||||
}
|
||||
if (process.env.NODECAST_LIVE_SEARCH_BROWSER === '1' && (size === 1200 || size === 100000)) {
|
||||
const chromium = require('playwright').chromium;
|
||||
const browser = await chromium.launch({ headless: true,
|
||||
executablePath: process.env.NODECAST_BROWSER_EXECUTABLE || undefined });
|
||||
try {
|
||||
const browserReport = await runBrowserJourney(browser, app.base, dir, size);
|
||||
assert.ok(browserReport.searches.length === BROWSER_TERMS.length);
|
||||
} finally { await browser.close(); }
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate a pre-FTS database on process restart. sqlite.js creates and backfills
|
||||
// both trigram indexes on the first catalog request after the server is ready.
|
||||
await stopServer(activeChild);
|
||||
activeChild = null;
|
||||
removeSearchIndexes(dir);
|
||||
app = await launchServer(t, dir);
|
||||
activeChild = app.child;
|
||||
const login = await fetch(`${app.base}/api/auth/login`, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ username: 'SearchPerf', password: 'password123' })
|
||||
});
|
||||
assert.equal(login.status, 200);
|
||||
const loginCookie = login.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0];
|
||||
const backfillStarted = performance.now();
|
||||
const firstAfterRestart = await apiRequest(app.base, loginCookie);
|
||||
const backfillMs = performance.now() - backfillStarted;
|
||||
assert.equal(firstAfterRestart.status, 200);
|
||||
const indexDb = new Database(path.join(dir, 'content.db'), { readonly: true });
|
||||
const indexes = indexDb.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'live_catalog_%_fts'`).all();
|
||||
const indexedPlaylistRows = indexDb.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n;
|
||||
indexDb.close();
|
||||
assert.equal(indexes.length, 2);
|
||||
assert.equal(indexedPlaylistRows, previousSize);
|
||||
console.log(`[live-search-diagnostic] ${JSON.stringify({ rows: previousSize, phase: 'first catalog request after dropping FTS/restart',
|
||||
httpMs: Number(firstAfterRestart.totalMs.toFixed(2)), backfillRequestMs: Number(backfillMs.toFixed(2)),
|
||||
bytes: firstAfterRestart.bytes, indexes: indexes.length, indexedPlaylistRows })}`);
|
||||
|
||||
const slow = reports.filter(report => report.httpMs >= SEARCH_LIMIT_MS);
|
||||
assert.deepEqual(slow, [], `search requests reached the observed 10s failure window: ${JSON.stringify(slow)}`);
|
||||
});
|
||||
175
test/live-catalog-search-shape-repro.test.js
Normal file
175
test/live-catalog-search-shape-repro.test.js
Normal file
@ -0,0 +1,175 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { performance } = require('node:perf_hooks');
|
||||
const Database = require('better-sqlite3');
|
||||
const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog');
|
||||
|
||||
const RUN = process.env.NODECAST_LIVE_SEARCH_SHAPE_REPRO === '1';
|
||||
const EXPECTED_LIVE = 38617;
|
||||
const EXPECTED_PLAYLIST = 66877;
|
||||
const EXPECTED_CATEGORIES = 1099;
|
||||
const INDEXED_LIMIT_MS = 1000;
|
||||
|
||||
function createFixtureDatabase() {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-shape-'));
|
||||
const db = new Database(path.join(dir, 'content.db'));
|
||||
db.pragma('journal_mode = OFF');
|
||||
db.exec(`
|
||||
CREATE TABLE categories (
|
||||
id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, category_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL, name TEXT NOT NULL, is_hidden INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE INDEX idx_categories_source_type ON categories(source_id, type);
|
||||
CREATE TABLE playlist_items (
|
||||
id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, item_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL, name TEXT NOT NULL, category_id TEXT, stream_url TEXT,
|
||||
is_hidden INTEGER DEFAULT 0, data JSON
|
||||
);
|
||||
CREATE INDEX idx_items_source_type ON playlist_items(source_id, type);
|
||||
CREATE INDEX idx_items_category ON playlist_items(source_id, category_id);
|
||||
CREATE INDEX idx_live_catalog_order ON playlist_items(name COLLATE NOCASE, source_id, id) WHERE type='live';
|
||||
CREATE TABLE favorites (
|
||||
id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, source_id INTEGER NOT NULL,
|
||||
item_id TEXT NOT NULL, item_type TEXT NOT NULL
|
||||
);
|
||||
`);
|
||||
|
||||
const insertCategory = db.prepare(`INSERT INTO categories (id, source_id, category_id, type, name, is_hidden)
|
||||
VALUES (?, 1, ?, 'live', ?, 0)`);
|
||||
const insertItem = db.prepare(`INSERT INTO playlist_items
|
||||
(id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data)
|
||||
VALUES (?, 1, ?, ?, ?, ?, 'http://fixture.invalid/stream', 0, ?)`);
|
||||
const typicalData = JSON.stringify({ padding: 'x'.repeat(370) });
|
||||
const largestData = JSON.stringify({ padding: 'x'.repeat(17870) });
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < EXPECTED_CATEGORIES; i++) {
|
||||
const categoryId = `group-${i}`;
|
||||
insertCategory.run(`1:${categoryId}`, categoryId,
|
||||
i < 5 ? `News Group ${i}` : `Bouquet ${i}`);
|
||||
}
|
||||
for (let i = 0; i < EXPECTED_PLAYLIST; i++) {
|
||||
const isLive = i < EXPECTED_LIVE;
|
||||
const offset = isLive ? i : i - EXPECTED_LIVE;
|
||||
const itemType = isLive ? 'live' : 'movie';
|
||||
const itemId = `${itemType}-${offset}`;
|
||||
const hasNews = isLive ? i < 549 : offset < 89;
|
||||
const name = hasNews ? `News ${isLive ? 'Channel' : 'Movie'} ${offset}`
|
||||
: `${isLive ? 'Channel' : 'Movie'} Catalog ${offset}`;
|
||||
const categoryId = `group-${offset % EXPECTED_CATEGORIES}`;
|
||||
insertItem.run(`1:${itemId}`, itemId, itemType, name, categoryId,
|
||||
i === 0 ? largestData : typicalData);
|
||||
}
|
||||
})();
|
||||
return { dir, db };
|
||||
}
|
||||
|
||||
function summarizePlan(planRows) {
|
||||
const tables = {};
|
||||
const indexes = [];
|
||||
let tempBtreeForOrder = false;
|
||||
let categoryCompositeLookup = false;
|
||||
for (const row of planRows) {
|
||||
const detail = String(row.detail || '').toUpperCase();
|
||||
if (detail.includes('USE TEMP B-TREE FOR ORDER BY')) tempBtreeForOrder = true;
|
||||
const match = detail.match(/(?:SCAN|SEARCH) ([A-Z_]+)/);
|
||||
if (!match) continue;
|
||||
const alias = match[1].toLowerCase();
|
||||
const indexMatch = detail.match(/USING (?:COVERING )?INDEX ([A-Z0-9_]+)/);
|
||||
if (indexMatch) indexes.push(indexMatch[1].toLowerCase());
|
||||
if (/SEARCH C USING INDEX IDX_CATEGORIES_SOURCE_TYPE_CATEGORY \(SOURCE_ID=\? AND TYPE=\? AND CATEGORY_ID=\?\)/.test(detail)) {
|
||||
categoryCompositeLookup = true;
|
||||
}
|
||||
tables[alias] ||= { scan: 0, search: 0, indexLookup: 0, virtualLookup: 0 };
|
||||
if (detail.startsWith('SCAN')) tables[alias].scan++;
|
||||
else tables[alias].search++;
|
||||
if (detail.includes('USING INDEX')) tables[alias].indexLookup++;
|
||||
if (detail.includes('VIRTUAL TABLE INDEX')) tables[alias].virtualLookup++;
|
||||
}
|
||||
return { steps: planRows.length, tables, indexes, categoryCompositeLookup, tempBtreeForOrder };
|
||||
}
|
||||
|
||||
function timedCatalog(db, q) {
|
||||
const options = parseQuery({ q, limit: '200' }, 1);
|
||||
let querySql;
|
||||
let queryParams;
|
||||
let sqlMs = 0;
|
||||
const prepare = db.prepare.bind(db);
|
||||
db.prepare = sql => {
|
||||
const statement = prepare(sql);
|
||||
if (!sql.includes('WITH live_sources AS')) return statement;
|
||||
return { all: (...args) => {
|
||||
querySql = sql;
|
||||
queryParams = args[0];
|
||||
const startedAt = performance.now();
|
||||
try { return statement.all(...args); }
|
||||
finally { sqlMs = performance.now() - startedAt; }
|
||||
} };
|
||||
};
|
||||
const startedAt = performance.now();
|
||||
const result = getLiveCatalog(db, [{ id: 1, type: 'm3u', enabled: true }], options);
|
||||
const serviceMs = performance.now() - startedAt;
|
||||
const plan = summarizePlan(prepare(`EXPLAIN QUERY PLAN ${querySql}`).all(queryParams));
|
||||
return { serviceMs: +serviceMs.toFixed(1), sqlMs: +sqlMs.toFixed(1),
|
||||
returned: result.items.length, hasMore: result.page.hasMore, plan };
|
||||
}
|
||||
|
||||
test('38k live rows and 1,099 groups stay fast with a composite category lookup index', {
|
||||
skip: RUN ? false : 'Opt-in local-shape performance reproduction',
|
||||
timeout: 180000
|
||||
}, t => {
|
||||
const { dir, db } = createFixtureDatabase();
|
||||
t.after(() => {
|
||||
db.close();
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
ensureSearch(db);
|
||||
const fixture = {
|
||||
liveRows: db.prepare("SELECT COUNT(*) AS n FROM playlist_items WHERE type='live'").get().n,
|
||||
playlistRows: db.prepare('SELECT COUNT(*) AS n FROM playlist_items').get().n,
|
||||
categoryRows: db.prepare('SELECT COUNT(*) AS n FROM categories').get().n,
|
||||
ftsPlaylistRows: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n,
|
||||
newsFtsCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?').get('%news%').n,
|
||||
liveNewsCandidates: db.prepare(`SELECT COUNT(*) AS n FROM playlist_items p WHERE p.type='live'
|
||||
AND p.rowid IN (SELECT rowid FROM live_catalog_playlist_items_fts WHERE name LIKE ?)`)
|
||||
.get('%news%').n,
|
||||
liveJson: db.prepare(`SELECT COUNT(*) AS n, SUM(length(CAST(data AS BLOB))) AS bytes,
|
||||
AVG(length(CAST(data AS BLOB))) AS avgBytes, MAX(length(CAST(data AS BLOB))) AS maxBytes
|
||||
FROM playlist_items WHERE type='live'`).get()
|
||||
};
|
||||
assert.deepEqual([fixture.liveRows, fixture.playlistRows, fixture.categoryRows, fixture.ftsPlaylistRows,
|
||||
fixture.newsFtsCandidates, fixture.liveNewsCandidates],
|
||||
[EXPECTED_LIVE, EXPECTED_PLAYLIST, EXPECTED_CATEGORIES, EXPECTED_PLAYLIST, 638, 549]);
|
||||
|
||||
const ftsStarted = performance.now();
|
||||
const ftsCandidates = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?')
|
||||
.get('%news%').n;
|
||||
const ftsMs = performance.now() - ftsStarted;
|
||||
const baseline = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') };
|
||||
for (const [query, measurement] of Object.entries(baseline)) {
|
||||
assert.equal(measurement.plan.categoryCompositeLookup, false,
|
||||
`baseline EXPLAIN QUERY PLAN for ${query} should not use the composite lookup index`);
|
||||
}
|
||||
|
||||
// Diagnostic-only control: add an index to the throwaway fixture, never the local DB.
|
||||
db.exec('CREATE INDEX idx_categories_source_type_category ON categories(source_id, type, category_id)');
|
||||
const withLookupIndex = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') };
|
||||
for (const [query, measurement] of Object.entries(withLookupIndex)) {
|
||||
assert.equal(measurement.plan.categoryCompositeLookup, true,
|
||||
`EXPLAIN QUERY PLAN for ${query} must use idx_categories_source_type_category on source_id, type, category_id`);
|
||||
}
|
||||
console.log(`[live-search-shape-repro] ${JSON.stringify({
|
||||
fixture: { ...fixture, liveJson: { rows: fixture.liveJson.n, bytes: fixture.liveJson.bytes,
|
||||
avgBytes: +fixture.liveJson.avgBytes.toFixed(1), maxBytes: fixture.liveJson.maxBytes },
|
||||
categoriesNameCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts WHERE name LIKE ?').get('%news%').n },
|
||||
ftsCandidateMs: +ftsMs.toFixed(2), baseline, withLookupIndex
|
||||
})}`);
|
||||
|
||||
for (const [query, measurement] of Object.entries(withLookupIndex)) {
|
||||
assert.ok(measurement.serviceMs < INDEXED_LIMIT_MS,
|
||||
`indexed catalogue route for ${query} took ${measurement.serviceMs}ms (limit ${INDEXED_LIMIT_MS}ms)`);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user