From 19849304a3843c902122da5af1532de62b73483c Mon Sep 17 00:00:00 2001 From: Blomios Date: Thu, 22 Oct 2026 23:23:46 +0200 Subject: [PATCH] fix(live): index category lookups for catalog search (#49) Avoid repeated category scans while filtering the paginated Live catalog, and keep the search timing and query-plan reproductions in the regression suite. Co-Authored-By: Claude Opus 4.8 --- server/db/sqlite.js | 2 + test/live-catalog-initial-load.test.js | 12 + test/live-catalog-search-performance.test.js | 360 +++++++++++++++++++ test/live-catalog-search-shape-repro.test.js | 175 +++++++++ 4 files changed, 549 insertions(+) create mode 100644 test/live-catalog-search-performance.test.js create mode 100644 test/live-catalog-search-shape-repro.test.js diff --git a/server/db/sqlite.js b/server/db/sqlite.js index d10c7e7..403e200 100644 --- a/server/db/sqlite.js +++ b/server/db/sqlite.js @@ -42,6 +42,8 @@ function initSchema() { data JSON -- Extra provider data ); CREATE INDEX IF NOT EXISTS idx_categories_source_type ON categories(source_id, type); + CREATE INDEX IF NOT EXISTS idx_categories_source_type_category + ON categories(source_id, type, category_id); `); // Playlist Items (Channels, Movies, Series, Episodes) diff --git a/test/live-catalog-initial-load.test.js b/test/live-catalog-initial-load.test.js index 6fefc5b..a0d6c27 100644 --- a/test/live-catalog-initial-load.test.js +++ b/test/live-catalog-initial-load.test.js @@ -77,6 +77,18 @@ async function requestAllPages(base, cookie, query = {}) { return items; } +test('application startup creates the composite category lookup index', async t => { + const app = await startApp(t); + const response = await request(app.base, app.cookie); + assert.equal(response.status, 200); + const db = new Database(path.join(app.dir, 'content.db'), { fileMustExist: true }); + t.after(() => db.close()); + const index = db.prepare(`SELECT sql FROM sqlite_master + WHERE type='index' AND name='idx_categories_source_type_category'`).get(); + assert.ok(index, 'startup schema should create idx_categories_source_type_category'); + assert.match(index.sql, /\(source_id,\s*type,\s*category_id\)/i); +}); + function seedCatalog(dir, userId) { const db = new Database(path.join(dir, 'content.db')); const insertCategory = db.prepare(` diff --git a/test/live-catalog-search-performance.test.js b/test/live-catalog-search-performance.test.js new file mode 100644 index 0000000..c06a087 --- /dev/null +++ b/test/live-catalog-search-performance.test.js @@ -0,0 +1,360 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { performance } = require('node:perf_hooks'); +const { spawn } = require('node:child_process'); +const Database = require('better-sqlite3'); +const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog'); + +const ROOT = path.join(__dirname, '..'); +const SCALE = process.env.NODECAST_LIVE_SEARCH_SIZES + ? process.env.NODECAST_LIVE_SEARCH_SIZES.split(',').map(Number) : [1200, 10000, 50000, 100000]; +const TERMS = ['news', 'sports', 'local', 'kids', 'world']; +const SEARCH_LIMIT_MS = 10000; +const BROWSER_TERMS = ['news', 'sports', 'local', 'kids', 'world']; + +async function launchServer(t, dir) { + const child = spawn(process.execPath, ['server/index.js'], { + cwd: ROOT, + env: { ...process.env, PORT: '0', NODECAST_DATA_DIR: dir, + NODECAST_DISABLE_BACKGROUND_JOBS: 'true', JWT_SECRET: '', SESSION_SECRET: '', + OIDC_ISSUER_URL: '', AUTH_COOKIE_SECURE: 'false' }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + t.after(async () => { + if (child.exitCode === null && child.signalCode === null) { + const closed = new Promise(resolve => child.once('close', resolve)); + child.kill(); + await closed; + } + }); + let output = ''; + const startedAt = performance.now(); + const base = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Startup timed out: ${output}`)), 30000); + child.stdout.on('data', chunk => { + output += chunk; + const match = output.match(/server running on http:\/\/localhost:(\d+)/i); + if (match) { clearTimeout(timer); resolve(`http://127.0.0.1:${match[1]}`); } + }); + child.stderr.on('data', chunk => { output += chunk; }); + child.once('exit', code => { + clearTimeout(timer); + reject(new Error(`Server exited ${code}: ${output}`)); + }); + }); + return { child, base, processReadyMs: performance.now() - startedAt }; +} + +async function stopServer(child) { + if (child.exitCode !== null || child.signalCode !== null) return; + const closed = new Promise(resolve => child.once('close', resolve)); + child.kill('SIGTERM'); + await closed; +} + +async function apiRequest(base, cookie, query = {}) { + const params = new URLSearchParams(query); + const start = performance.now(); + const response = await fetch(`${base}/api/channels/live-catalog?${params}`, { + headers: { Cookie: cookie } + }); + const text = await response.text(); + return { status: response.status, totalMs: performance.now() - start, + bytes: Buffer.byteLength(text), body: JSON.parse(text) }; +} + +function seedDelta(dir, from, to) { + const db = new Database(path.join(dir, 'content.db')); + const insertCategory = db.prepare(` + INSERT OR IGNORE INTO categories (id, source_id, category_id, type, name, is_hidden) + VALUES (?, 1, ?, 'live', ?, 0) + `); + const insertItem = db.prepare(` + INSERT INTO playlist_items (id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data) + VALUES (?, 1, ?, 'live', ?, ?, ?, 0, '{}') + `); + const themes = ['News', 'Sports', 'Local', 'Kids', 'World']; + const groupCount = 20; + const insert = db.transaction(() => { + for (let group = 0; group < groupCount; group++) { + const categoryId = `bouquet-${String(group).padStart(2, '0')}`; + insertCategory.run(`1:${categoryId}`, categoryId, `Bouquet ${String(group + 1).padStart(2, '0')}`); + } + for (let index = from; index < to; index++) { + const id = `live-${String(index).padStart(6, '0')}`; + const theme = themes[index % themes.length]; + const categoryId = `bouquet-${String(index % groupCount).padStart(2, '0')}`; + const region = `Region ${String(index % 12 + 1).padStart(2, '0')}`; + insertItem.run(`1:${id}`, id, `${theme} ${region} Channel ${String(index).padStart(6, '0')} HD`, + categoryId, `http://streams.test/${id}`); + } + }); + insert(); + db.close(); +} + +function removeSearchIndexes(dir) { + const db = new Database(path.join(dir, 'content.db')); + db.exec(`DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ai; + DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_ad; + DROP TRIGGER IF EXISTS live_catalog_playlist_items_fts_au; + DROP TRIGGER IF EXISTS live_catalog_categories_fts_ai; + DROP TRIGGER IF EXISTS live_catalog_categories_fts_ad; + DROP TRIGGER IF EXISTS live_catalog_categories_fts_au; + DROP TABLE IF EXISTS live_catalog_playlist_items_fts; + DROP TABLE IF EXISTS live_catalog_categories_fts;`); + db.close(); +} + +function searchIndexSnapshot(dir) { + const db = new Database(path.join(dir, 'content.db'), { readonly: true }); + const schemaVersion = db.pragma('schema_version', { simple: true }); + const definitions = db.prepare(`SELECT name, sql FROM sqlite_master WHERE name LIKE 'live_catalog_%' + ORDER BY name`).all(); + const playlistRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n; + const categoryRows = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts').get().n; + db.close(); + return { schemaVersion, definitions, playlistRows, categoryRows }; +} + +function endpointKind(url) { + if (url.pathname === '/api/channels/live-catalog') return 'live-catalog'; + if (url.pathname === '/api/channels/hidden') return 'channels/hidden'; + if (url.pathname === '/api/favorites') return 'favorites'; + if (url.pathname.startsWith('/api/proxy/epg/')) return 'epg'; + if (url.pathname === '/api/history') return 'history'; + return null; +} + +async function runBrowserJourney(chromium, base, dir, size) { + const context = await chromium.newContext(); + const page = await context.newPage(); + const requests = []; + const byRequest = new Map(); + const responseTasks = []; + page.on('request', request => { + const url = new URL(request.url()); + const kind = endpointKind(url); + if (!kind) return; + const entry = { kind, method: request.method(), path: url.pathname, + query: Object.fromEntries(url.searchParams), startedAt: performance.now() }; + byRequest.set(request, entry); + requests.push(entry); + }); + page.on('response', response => { + const entry = byRequest.get(response.request()); + if (!entry) return; + entry.status = response.status(); + entry.durationMs = Number((performance.now() - entry.startedAt).toFixed(2)); + const task = response.body().then(body => { entry.responseBytes = body.length; }, () => { entry.responseBytes = null; }); + responseTasks.push(task); + }); + await page.route('**/api/proxy/epg/**', route => route.fulfill({ + status: 200, contentType: 'application/json', + body: JSON.stringify({ channels: [], programmes: [] }) + })); + await page.route('https://cdn.jsdelivr.net/npm/hls.js@1.5.7/dist/hls.min.js', route => route.abort()); + await page.route('https://fonts.**', route => route.abort()); + + const blankContext = { cookies: (await context.cookies()).length, + localStorage: await page.evaluate(() => Object.keys(localStorage).length).catch(() => 0) }; + const homeStarted = performance.now(); + await page.goto(`${base}/login.html`); + await page.locator('#username').fill('SearchPerf'); + await page.locator('#password').fill('password123'); + await page.locator('#password').press('Enter'); + await page.waitForURL(url => url.pathname === '/'); + await page.waitForFunction(() => window.app?.currentPage === 'home' + && document.querySelector('#favorite-channels-list') + && !document.querySelector('#favorite-channels-list .loading-state')); + await page.waitForFunction(() => window.app?.epgGuide?.lastRefreshTime || + document.querySelector('#epg-container')?.textContent?.includes('No EPG')); + const homeReadyMs = performance.now() - homeStarted; + const homeNetwork = requests.slice(); + + const liveStarted = performance.now(); + const liveRequestIndex = requests.length; + await page.locator('.nav-link[data-page="live"]').click(); + await page.waitForFunction(() => window.app?.currentPage === 'live' + && window.app.channelList.channels.length === 200 + && !window.app.channelList.isLoading + && document.querySelector('#channel-list')?.getAttribute('aria-busy') === 'false'); + const liveReadyMs = performance.now() - liveStarted; + const initialLiveNetwork = requests.slice(liveRequestIndex); + + const beforeIndexes = searchIndexSnapshot(dir); + const searches = []; + for (const term of BROWSER_TERMS) { + const firstRequest = requests.length; + const startedAt = performance.now(); + await page.locator('#channel-search').fill(term); + try { await page.waitForFunction(value => { + const list = window.app?.channelList; + const container = document.querySelector('#channel-list'); + return list?.catalogQuery === value && !list.isLoading && list.channels.length === 200 + && container?.getAttribute('aria-busy') === 'false'; + }, term, { timeout: 5000 }); } catch (error) { + const browserState = await page.evaluate(() => ({ currentPage: window.app?.currentPage, + searchValue: document.querySelector('#channel-search')?.value, + catalogQuery: window.app?.channelList?.catalogQuery, + channelCount: window.app?.channelList?.channels?.length, + isLoading: window.app?.channelList?.isLoading, + status: document.querySelector('#channel-list')?.getAttribute('aria-busy'), + groups: window.app?.channelList?.sortedGroups?.length })); + console.log(`[live-search-browser-debug] ${JSON.stringify({ size, term, browserState, + requests: requests.slice(firstRequest).map(entry => ({ kind: entry.kind, query: entry.query, status: entry.status })) })}`); + throw error; + } + await page.waitForTimeout(75); + const settledAt = performance.now(); + const current = requests.slice(firstRequest); + await Promise.all(responseTasks); + const indexState = searchIndexSnapshot(dir); + searches.push({ term, inputToStableDomMs: Number((settledAt - startedAt).toFixed(2)), + channels: await page.evaluate(() => window.app.channelList.channels.length), + domChannelGroups: await page.locator('#channel-list .channel-group').count(), + network: current.map(entry => ({ endpoint: entry.kind, method: entry.method, + query: entry.query, status: entry.status, responseBytes: entry.responseBytes, + durationMs: entry.durationMs })), + ftsSchemaVersion: indexState.schemaVersion, + ftsRows: { playlist: indexState.playlistRows, categories: indexState.categoryRows }, + schemaUnchanged: indexState.schemaVersion === beforeIndexes.schemaVersion + && JSON.stringify(indexState.definitions) === JSON.stringify(beforeIndexes.definitions) + && indexState.playlistRows === beforeIndexes.playlistRows + && indexState.categoryRows === beforeIndexes.categoryRows }); + } + await Promise.all(responseTasks); + const afterIndexes = searchIndexSnapshot(dir); + const report = { rows: size, blankContext: blankContext.cookies === 0 && blankContext.localStorage === 0, + homeReadyMs: Number(homeReadyMs.toFixed(2)), liveReadyMs: Number(liveReadyMs.toFixed(2)), + homeNetwork: homeNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method, + query: entry.query, status: entry.status, responseBytes: entry.responseBytes, + durationMs: entry.durationMs })), + initialLiveNetwork: initialLiveNetwork.map(entry => ({ endpoint: entry.kind, method: entry.method, + query: entry.query, status: entry.status, responseBytes: entry.responseBytes, + durationMs: entry.durationMs })), + searches, ftsUnchangedAcrossSearches: JSON.stringify(beforeIndexes) === JSON.stringify(afterIndexes) }; + console.log(`[live-search-browser] ${JSON.stringify(report)}`); + assert.equal(report.blankContext, true, 'the browser must start without cookies or localStorage'); + assert.ok(searches.every(search => search.schemaUnchanged), 'search requests must not recreate or backfill FTS indexes'); + assert.equal(report.ftsUnchangedAcrossSearches, true); + await context.close(); + return report; +} + +function measureService(dir, userId, term) { + const db = new Database(path.join(dir, 'content.db')); + ensureSearch(db); + let sqlMs = 0; + const prepare = db.prepare.bind(db); + db.prepare = sql => { + const statement = prepare(sql); + if (!sql.includes('WITH live_sources AS')) return statement; + return { all: (...args) => { + const startedAt = performance.now(); + try { return statement.all(...args); } + finally { sqlMs += performance.now() - startedAt; } + } }; + }; + const allSources = [{ id: 1, type: 'm3u', enabled: true }]; + const options = parseQuery({ q: term, limit: '100' }, userId); + const startedAt = performance.now(); + const result = getLiveCatalog(db, allSources, options); + const serviceMs = performance.now() - startedAt; + db.close(); + return { sqlMs, serviceMs, returned: result.items.length, hasMore: result.page.hasMore }; +} + +test('live catalog repeated searches record first-use FTS backfill and per-query timing at 1.2k–100k rows', { + skip: process.env.NODECAST_LIVE_SEARCH_DIAGNOSTIC === '1' ? false : 'Opt-in diagnostic benchmark', + timeout: 300000 +}, async t => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-perf-')); + fs.writeFileSync(path.join(dir, 'db.json'), JSON.stringify({ + sources: [ + { id: 1, type: 'm3u', name: 'Fixture', url: 'http://127.0.0.1:1/unused.m3u', enabled: true }, + { id: 2, type: 'epg', name: 'Fixture EPG', url: 'http://epg.fixture.invalid/guide.xml', enabled: true } + ], + hiddenItems: [], favorites: [], users: [], nextId: 3 + })); + let activeChild; + t.after(async () => { + if (activeChild) await stopServer(activeChild); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + let app = await launchServer(t, dir); + activeChild = app.child; + const setup = await fetch(`${app.base}/api/auth/setup`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username: 'SearchPerf', password: 'password123' }) + }); + assert.equal(setup.status, 201); + const setupBody = await setup.json(); + const cookie = setup.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0]; + const initialized = await apiRequest(app.base, cookie); + assert.equal(initialized.status, 200); + + const reports = []; + let previousSize = 0; + for (const size of SCALE) { + seedDelta(dir, previousSize, size); + previousSize = size; + for (const [round, term] of [...TERMS, ...TERMS].entries()) { + const response = await apiRequest(app.base, cookie, { q: term }); + assert.equal(response.status, 200); + assert.equal(response.body.items.length, 100); + assert.equal(response.body.page.hasMore, true); + const sql = measureService(dir, setupBody.user.id, term); + const expectedMatches = Math.floor((size - 1 - TERMS.indexOf(term)) / TERMS.length) + 1; + const report = { rows: size, phase: round < TERMS.length ? 'first distinct pass' : 'repeat pass', + term, expectedMatches, returned: response.body.items.length, hasMore: response.body.page.hasMore, + bytes: response.bytes, httpMs: Number(response.totalMs.toFixed(2)), + sqlMs: Number(sql.sqlMs.toFixed(2)), serviceMs: Number(sql.serviceMs.toFixed(2)) }; + reports.push(report); + console.log(`[live-search-diagnostic] ${JSON.stringify(report)}`); + } + if (process.env.NODECAST_LIVE_SEARCH_BROWSER === '1' && (size === 1200 || size === 100000)) { + const chromium = require('playwright').chromium; + const browser = await chromium.launch({ headless: true, + executablePath: process.env.NODECAST_BROWSER_EXECUTABLE || undefined }); + try { + const browserReport = await runBrowserJourney(browser, app.base, dir, size); + assert.ok(browserReport.searches.length === BROWSER_TERMS.length); + } finally { await browser.close(); } + } + } + + // Simulate a pre-FTS database on process restart. sqlite.js creates and backfills + // both trigram indexes on the first catalog request after the server is ready. + await stopServer(activeChild); + activeChild = null; + removeSearchIndexes(dir); + app = await launchServer(t, dir); + activeChild = app.child; + const login = await fetch(`${app.base}/api/auth/login`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username: 'SearchPerf', password: 'password123' }) + }); + assert.equal(login.status, 200); + const loginCookie = login.headers.getSetCookie().find(value => value.startsWith('nodecast_auth_')).split(';')[0]; + const backfillStarted = performance.now(); + const firstAfterRestart = await apiRequest(app.base, loginCookie); + const backfillMs = performance.now() - backfillStarted; + assert.equal(firstAfterRestart.status, 200); + const indexDb = new Database(path.join(dir, 'content.db'), { readonly: true }); + const indexes = indexDb.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'live_catalog_%_fts'`).all(); + const indexedPlaylistRows = indexDb.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n; + indexDb.close(); + assert.equal(indexes.length, 2); + assert.equal(indexedPlaylistRows, previousSize); + console.log(`[live-search-diagnostic] ${JSON.stringify({ rows: previousSize, phase: 'first catalog request after dropping FTS/restart', + httpMs: Number(firstAfterRestart.totalMs.toFixed(2)), backfillRequestMs: Number(backfillMs.toFixed(2)), + bytes: firstAfterRestart.bytes, indexes: indexes.length, indexedPlaylistRows })}`); + + const slow = reports.filter(report => report.httpMs >= SEARCH_LIMIT_MS); + assert.deepEqual(slow, [], `search requests reached the observed 10s failure window: ${JSON.stringify(slow)}`); +}); diff --git a/test/live-catalog-search-shape-repro.test.js b/test/live-catalog-search-shape-repro.test.js new file mode 100644 index 0000000..0679317 --- /dev/null +++ b/test/live-catalog-search-shape-repro.test.js @@ -0,0 +1,175 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); +const Database = require('better-sqlite3'); +const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog'); + +const RUN = process.env.NODECAST_LIVE_SEARCH_SHAPE_REPRO === '1'; +const EXPECTED_LIVE = 38617; +const EXPECTED_PLAYLIST = 66877; +const EXPECTED_CATEGORIES = 1099; +const INDEXED_LIMIT_MS = 1000; + +function createFixtureDatabase() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-shape-')); + const db = new Database(path.join(dir, 'content.db')); + db.pragma('journal_mode = OFF'); + db.exec(` + CREATE TABLE categories ( + id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, category_id TEXT NOT NULL, + type TEXT NOT NULL, name TEXT NOT NULL, is_hidden INTEGER DEFAULT 0 + ); + CREATE INDEX idx_categories_source_type ON categories(source_id, type); + CREATE TABLE playlist_items ( + id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, item_id TEXT NOT NULL, + type TEXT NOT NULL, name TEXT NOT NULL, category_id TEXT, stream_url TEXT, + is_hidden INTEGER DEFAULT 0, data JSON + ); + CREATE INDEX idx_items_source_type ON playlist_items(source_id, type); + CREATE INDEX idx_items_category ON playlist_items(source_id, category_id); + CREATE INDEX idx_live_catalog_order ON playlist_items(name COLLATE NOCASE, source_id, id) WHERE type='live'; + CREATE TABLE favorites ( + id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, item_type TEXT NOT NULL + ); + `); + + const insertCategory = db.prepare(`INSERT INTO categories (id, source_id, category_id, type, name, is_hidden) + VALUES (?, 1, ?, 'live', ?, 0)`); + const insertItem = db.prepare(`INSERT INTO playlist_items + (id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data) + VALUES (?, 1, ?, ?, ?, ?, 'http://fixture.invalid/stream', 0, ?)`); + const typicalData = JSON.stringify({ padding: 'x'.repeat(370) }); + const largestData = JSON.stringify({ padding: 'x'.repeat(17870) }); + db.transaction(() => { + for (let i = 0; i < EXPECTED_CATEGORIES; i++) { + const categoryId = `group-${i}`; + insertCategory.run(`1:${categoryId}`, categoryId, + i < 5 ? `News Group ${i}` : `Bouquet ${i}`); + } + for (let i = 0; i < EXPECTED_PLAYLIST; i++) { + const isLive = i < EXPECTED_LIVE; + const offset = isLive ? i : i - EXPECTED_LIVE; + const itemType = isLive ? 'live' : 'movie'; + const itemId = `${itemType}-${offset}`; + const hasNews = isLive ? i < 549 : offset < 89; + const name = hasNews ? `News ${isLive ? 'Channel' : 'Movie'} ${offset}` + : `${isLive ? 'Channel' : 'Movie'} Catalog ${offset}`; + const categoryId = `group-${offset % EXPECTED_CATEGORIES}`; + insertItem.run(`1:${itemId}`, itemId, itemType, name, categoryId, + i === 0 ? largestData : typicalData); + } + })(); + return { dir, db }; +} + +function summarizePlan(planRows) { + const tables = {}; + const indexes = []; + let tempBtreeForOrder = false; + let categoryCompositeLookup = false; + for (const row of planRows) { + const detail = String(row.detail || '').toUpperCase(); + if (detail.includes('USE TEMP B-TREE FOR ORDER BY')) tempBtreeForOrder = true; + const match = detail.match(/(?:SCAN|SEARCH) ([A-Z_]+)/); + if (!match) continue; + const alias = match[1].toLowerCase(); + const indexMatch = detail.match(/USING (?:COVERING )?INDEX ([A-Z0-9_]+)/); + if (indexMatch) indexes.push(indexMatch[1].toLowerCase()); + if (/SEARCH C USING INDEX IDX_CATEGORIES_SOURCE_TYPE_CATEGORY \(SOURCE_ID=\? AND TYPE=\? AND CATEGORY_ID=\?\)/.test(detail)) { + categoryCompositeLookup = true; + } + tables[alias] ||= { scan: 0, search: 0, indexLookup: 0, virtualLookup: 0 }; + if (detail.startsWith('SCAN')) tables[alias].scan++; + else tables[alias].search++; + if (detail.includes('USING INDEX')) tables[alias].indexLookup++; + if (detail.includes('VIRTUAL TABLE INDEX')) tables[alias].virtualLookup++; + } + return { steps: planRows.length, tables, indexes, categoryCompositeLookup, tempBtreeForOrder }; +} + +function timedCatalog(db, q) { + const options = parseQuery({ q, limit: '200' }, 1); + let querySql; + let queryParams; + let sqlMs = 0; + const prepare = db.prepare.bind(db); + db.prepare = sql => { + const statement = prepare(sql); + if (!sql.includes('WITH live_sources AS')) return statement; + return { all: (...args) => { + querySql = sql; + queryParams = args[0]; + const startedAt = performance.now(); + try { return statement.all(...args); } + finally { sqlMs = performance.now() - startedAt; } + } }; + }; + const startedAt = performance.now(); + const result = getLiveCatalog(db, [{ id: 1, type: 'm3u', enabled: true }], options); + const serviceMs = performance.now() - startedAt; + const plan = summarizePlan(prepare(`EXPLAIN QUERY PLAN ${querySql}`).all(queryParams)); + return { serviceMs: +serviceMs.toFixed(1), sqlMs: +sqlMs.toFixed(1), + returned: result.items.length, hasMore: result.page.hasMore, plan }; +} + +test('38k live rows and 1,099 groups stay fast with a composite category lookup index', { + skip: RUN ? false : 'Opt-in local-shape performance reproduction', + timeout: 180000 +}, t => { + const { dir, db } = createFixtureDatabase(); + t.after(() => { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + ensureSearch(db); + const fixture = { + liveRows: db.prepare("SELECT COUNT(*) AS n FROM playlist_items WHERE type='live'").get().n, + playlistRows: db.prepare('SELECT COUNT(*) AS n FROM playlist_items').get().n, + categoryRows: db.prepare('SELECT COUNT(*) AS n FROM categories').get().n, + ftsPlaylistRows: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n, + newsFtsCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?').get('%news%').n, + liveNewsCandidates: db.prepare(`SELECT COUNT(*) AS n FROM playlist_items p WHERE p.type='live' + AND p.rowid IN (SELECT rowid FROM live_catalog_playlist_items_fts WHERE name LIKE ?)`) + .get('%news%').n, + liveJson: db.prepare(`SELECT COUNT(*) AS n, SUM(length(CAST(data AS BLOB))) AS bytes, + AVG(length(CAST(data AS BLOB))) AS avgBytes, MAX(length(CAST(data AS BLOB))) AS maxBytes + FROM playlist_items WHERE type='live'`).get() + }; + assert.deepEqual([fixture.liveRows, fixture.playlistRows, fixture.categoryRows, fixture.ftsPlaylistRows, + fixture.newsFtsCandidates, fixture.liveNewsCandidates], + [EXPECTED_LIVE, EXPECTED_PLAYLIST, EXPECTED_CATEGORIES, EXPECTED_PLAYLIST, 638, 549]); + + const ftsStarted = performance.now(); + const ftsCandidates = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?') + .get('%news%').n; + const ftsMs = performance.now() - ftsStarted; + const baseline = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') }; + for (const [query, measurement] of Object.entries(baseline)) { + assert.equal(measurement.plan.categoryCompositeLookup, false, + `baseline EXPLAIN QUERY PLAN for ${query} should not use the composite lookup index`); + } + + // Diagnostic-only control: add an index to the throwaway fixture, never the local DB. + db.exec('CREATE INDEX idx_categories_source_type_category ON categories(source_id, type, category_id)'); + const withLookupIndex = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') }; + for (const [query, measurement] of Object.entries(withLookupIndex)) { + assert.equal(measurement.plan.categoryCompositeLookup, true, + `EXPLAIN QUERY PLAN for ${query} must use idx_categories_source_type_category on source_id, type, category_id`); + } + console.log(`[live-search-shape-repro] ${JSON.stringify({ + fixture: { ...fixture, liveJson: { rows: fixture.liveJson.n, bytes: fixture.liveJson.bytes, + avgBytes: +fixture.liveJson.avgBytes.toFixed(1), maxBytes: fixture.liveJson.maxBytes }, + categoriesNameCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts WHERE name LIKE ?').get('%news%').n }, + ftsCandidateMs: +ftsMs.toFixed(2), baseline, withLookupIndex + })}`); + + for (const [query, measurement] of Object.entries(withLookupIndex)) { + assert.ok(measurement.serviceMs < INDEXED_LIMIT_MS, + `indexed catalogue route for ${query} took ${measurement.serviceMs}ms (limit ${INDEXED_LIMIT_MS}ms)`); + } +});