NodecastV2/test/live-catalog-search-shape-repro.test.js

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { performance } = require('node:perf_hooks');
const Database = require('better-sqlite3');
const { parseQuery, getLiveCatalog, ensureSearch } = require('../server/services/liveCatalog');

const RUN = process.env.NODECAST_LIVE_SEARCH_SHAPE_REPRO === '1';
const EXPECTED_LIVE = 38617;
const EXPECTED_PLAYLIST = 66877;
const EXPECTED_CATEGORIES = 1099;
const INDEXED_LIMIT_MS = 1000;

function createFixtureDatabase() {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nodecast-live-search-shape-'));
    const db = new Database(path.join(dir, 'content.db'));
    db.pragma('journal_mode = OFF');
    db.exec(`
        CREATE TABLE categories (
            id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, category_id TEXT NOT NULL,
            type TEXT NOT NULL, name TEXT NOT NULL, is_hidden INTEGER DEFAULT 0
        );
        CREATE INDEX idx_categories_source_type ON categories(source_id, type);
        CREATE TABLE playlist_items (
            id TEXT PRIMARY KEY, source_id INTEGER NOT NULL, item_id TEXT NOT NULL,
            type TEXT NOT NULL, name TEXT NOT NULL, category_id TEXT, stream_url TEXT,
            is_hidden INTEGER DEFAULT 0, data JSON
        );
        CREATE INDEX idx_items_source_type ON playlist_items(source_id, type);
        CREATE INDEX idx_items_category ON playlist_items(source_id, category_id);
        CREATE INDEX idx_live_catalog_order ON playlist_items(name COLLATE NOCASE, source_id, id) WHERE type='live';
        CREATE TABLE favorites (
            id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, source_id INTEGER NOT NULL,
            item_id TEXT NOT NULL, item_type TEXT NOT NULL
        );
    `);

    const insertCategory = db.prepare(`INSERT INTO categories (id, source_id, category_id, type, name, is_hidden)
        VALUES (?, 1, ?, 'live', ?, 0)`);
    const insertItem = db.prepare(`INSERT INTO playlist_items
        (id, source_id, item_id, type, name, category_id, stream_url, is_hidden, data)
        VALUES (?, 1, ?, ?, ?, ?, 'http://fixture.invalid/stream', 0, ?)`);
    const typicalData = JSON.stringify({ padding: 'x'.repeat(370) });
    const largestData = JSON.stringify({ padding: 'x'.repeat(17870) });
    db.transaction(() => {
        for (let i = 0; i < EXPECTED_CATEGORIES; i++) {
            const categoryId = `group-${i}`;
            insertCategory.run(`1:${categoryId}`, categoryId,
                i < 5 ? `News Group ${i}` : `Bouquet ${i}`);
        }
        for (let i = 0; i < EXPECTED_PLAYLIST; i++) {
            const isLive = i < EXPECTED_LIVE;
            const offset = isLive ? i : i - EXPECTED_LIVE;
            const itemType = isLive ? 'live' : 'movie';
            const itemId = `${itemType}-${offset}`;
            const hasNews = isLive ? i < 549 : offset < 89;
            const name = hasNews ? `News ${isLive ? 'Channel' : 'Movie'} ${offset}`
                : `${isLive ? 'Channel' : 'Movie'} Catalog ${offset}`;
            const categoryId = `group-${offset % EXPECTED_CATEGORIES}`;
            insertItem.run(`1:${itemId}`, itemId, itemType, name, categoryId,
                i === 0 ? largestData : typicalData);
        }
    })();
    return { dir, db };
}

function summarizePlan(planRows) {
    const tables = {};
    const indexes = [];
    let tempBtreeForOrder = false;
    let categoryCompositeLookup = false;
    for (const row of planRows) {
        const detail = String(row.detail || '').toUpperCase();
        if (detail.includes('USE TEMP B-TREE FOR ORDER BY')) tempBtreeForOrder = true;
        const match = detail.match(/(?:SCAN|SEARCH) ([A-Z_]+)/);
        if (!match) continue;
        const alias = match[1].toLowerCase();
        const indexMatch = detail.match(/USING (?:COVERING )?INDEX ([A-Z0-9_]+)/);
        if (indexMatch) indexes.push(indexMatch[1].toLowerCase());
        if (/SEARCH C USING INDEX IDX_CATEGORIES_SOURCE_TYPE_CATEGORY \(SOURCE_ID=\? AND TYPE=\? AND CATEGORY_ID=\?\)/.test(detail)) {
            categoryCompositeLookup = true;
        }
        tables[alias] ||= { scan: 0, search: 0, indexLookup: 0, virtualLookup: 0 };
        if (detail.startsWith('SCAN')) tables[alias].scan++;
        else tables[alias].search++;
        if (detail.includes('USING INDEX')) tables[alias].indexLookup++;
        if (detail.includes('VIRTUAL TABLE INDEX')) tables[alias].virtualLookup++;
    }
    return { steps: planRows.length, tables, indexes, categoryCompositeLookup, tempBtreeForOrder };
}

function timedCatalog(db, q) {
    const options = parseQuery({ q, limit: '200' }, 1);
    let querySql;
    let queryParams;
    let sqlMs = 0;
    const prepare = db.prepare.bind(db);
    db.prepare = sql => {
        const statement = prepare(sql);
        if (!sql.includes('WITH live_sources AS')) return statement;
        return { all: (...args) => {
            querySql = sql;
            queryParams = args[0];
            const startedAt = performance.now();
            try { return statement.all(...args); }
            finally { sqlMs = performance.now() - startedAt; }
        } };
    };
    const startedAt = performance.now();
    const result = getLiveCatalog(db, [{ id: 1, type: 'm3u', enabled: true }], options);
    const serviceMs = performance.now() - startedAt;
    const plan = summarizePlan(prepare(`EXPLAIN QUERY PLAN ${querySql}`).all(queryParams));
    return { serviceMs: +serviceMs.toFixed(1), sqlMs: +sqlMs.toFixed(1),
        returned: result.items.length, hasMore: result.page.hasMore, plan };
}

test('38k live rows and 1,099 groups stay fast with a composite category lookup index', {
    skip: RUN ? false : 'Opt-in local-shape performance reproduction',
    timeout: 180000
}, t => {
    const { dir, db } = createFixtureDatabase();
    t.after(() => {
        db.close();
        fs.rmSync(dir, { recursive: true, force: true });
    });

    ensureSearch(db);
    const fixture = {
        liveRows: db.prepare("SELECT COUNT(*) AS n FROM playlist_items WHERE type='live'").get().n,
        playlistRows: db.prepare('SELECT COUNT(*) AS n FROM playlist_items').get().n,
        categoryRows: db.prepare('SELECT COUNT(*) AS n FROM categories').get().n,
        ftsPlaylistRows: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts').get().n,
        newsFtsCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?').get('%news%').n,
        liveNewsCandidates: db.prepare(`SELECT COUNT(*) AS n FROM playlist_items p WHERE p.type='live'
            AND p.rowid IN (SELECT rowid FROM live_catalog_playlist_items_fts WHERE name LIKE ?)`)
            .get('%news%').n,
        liveJson: db.prepare(`SELECT COUNT(*) AS n, SUM(length(CAST(data AS BLOB))) AS bytes,
            AVG(length(CAST(data AS BLOB))) AS avgBytes, MAX(length(CAST(data AS BLOB))) AS maxBytes
            FROM playlist_items WHERE type='live'`).get()
    };
    assert.deepEqual([fixture.liveRows, fixture.playlistRows, fixture.categoryRows, fixture.ftsPlaylistRows,
        fixture.newsFtsCandidates, fixture.liveNewsCandidates],
    [EXPECTED_LIVE, EXPECTED_PLAYLIST, EXPECTED_CATEGORIES, EXPECTED_PLAYLIST, 638, 549]);

    const ftsStarted = performance.now();
    const ftsCandidates = db.prepare('SELECT COUNT(*) AS n FROM live_catalog_playlist_items_fts WHERE name LIKE ?')
        .get('%news%').n;
    const ftsMs = performance.now() - ftsStarted;
    const baseline = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') };
    for (const [query, measurement] of Object.entries(baseline)) {
        assert.equal(measurement.plan.categoryCompositeLookup, false,
            `baseline EXPLAIN QUERY PLAN for ${query} should not use the composite lookup index`);
    }

    // Diagnostic-only control: add an index to the throwaway fixture, never the local DB.
    db.exec('CREATE INDEX idx_categories_source_type_category ON categories(source_id, type, category_id)');
    const withLookupIndex = { noQuery: timedCatalog(db, ''), news: timedCatalog(db, 'news') };
    for (const [query, measurement] of Object.entries(withLookupIndex)) {
        assert.equal(measurement.plan.categoryCompositeLookup, true,
            `EXPLAIN QUERY PLAN for ${query} must use idx_categories_source_type_category on source_id, type, category_id`);
    }
    console.log(`[live-search-shape-repro] ${JSON.stringify({
        fixture: { ...fixture, liveJson: { rows: fixture.liveJson.n, bytes: fixture.liveJson.bytes,
            avgBytes: +fixture.liveJson.avgBytes.toFixed(1), maxBytes: fixture.liveJson.maxBytes },
            categoriesNameCandidates: db.prepare('SELECT COUNT(*) AS n FROM live_catalog_categories_fts WHERE name LIKE ?').get('%news%').n },
        ftsCandidateMs: +ftsMs.toFixed(2), baseline, withLookupIndex
    })}`);

    for (const [query, measurement] of Object.entries(withLookupIndex)) {
        assert.ok(measurement.serviceMs < INDEXED_LIMIT_MS,
            `indexed catalogue route for ${query} took ${measurement.serviceMs}ms (limit ${INDEXED_LIMIT_MS}ms)`);
    }
});