feat: persist incremental live channel analysis

Persist live channel language metadata across background projection rebuilds and target Local AI identity analysis to newly detected streams unless a manual full analysis is requested.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

test/liveAnalysisUserFlowDom.test.js

@@ -141,12 +141,12 @@ test('Settings renders Laya batch progress and removes misleading values when it
     assert.equal(bar.style.width, '');
 });
 
-test('analysis remains visibly busy while Laya language batches run after projection completes', async () => {
+test('analysis completion reloads the persisted projection', async () => {
     const { window } = createDom();
     let rebuildClicked = false;
-    let languageRefreshStarted = false;
-    let releaseLanguageRefresh;
-    const languageRefresh = new Promise(resolve => { releaseLanguageRefresh = resolve; });
+    let projectionReloadStarted = false;
+    let releaseProjectionReload;
+    const projectionReload = new Promise(resolve => { releaseProjectionReload = resolve; });
     window.API = {
         live: {
             getProjectionStatus: async () => rebuildClicked
@@ -176,11 +176,11 @@ test('analysis remains visibly busy while Laya language batches run after projec
             saveSettings() {}
         },
         channelList: {
-            loadChannels: async () => {},
-            refreshLayaLanguages: async () => {
-                languageRefreshStarted = true;
-                await languageRefresh;
-            }
+            loadChannels: async () => {
+                projectionReloadStarted = true;
+                await projectionReload;
+            },
+            refreshLayaLanguages: async () => assert.fail('Browser-side Laya refresh must not run')
         }
     };
     window.app = app;
@@ -193,28 +193,19 @@ test('analysis remains visibly busy while Laya language batches run after projec
     window.document.getElementById('live-analysis-button').click();
 
     try {
-        await waitFor(() => languageRefreshStarted, 'Expected Laya language refresh to start after projection completion');
-        assert.notEqual(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete');
-        assert.equal(window.document.getElementById('live-analysis-progress').hidden, false);
+        await waitFor(() => projectionReloadStarted, 'Expected persisted projection reload after analysis completion');
+        assert.equal(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete');
     } finally {
-        releaseLanguageRefresh();
+        releaseProjectionReload();
         window.clearInterval(settingsPage.liveAnalysisPollTimer);
     }
 });
 
-test('user analysis click keeps visible channels after Local AI language failure', async () => {
+test('user analysis click reloads persisted languages without a browser-side Laya batch', async () => {
     const { window } = createDom();
     let rebuildClicked = 0;
     let layaBatchCalls = 0;
     let channelLoads = 0;
-    const consoleWarnings = [];
-    window.console = {
-        ...console,
-        warn: (...args) => {
-            consoleWarnings.push(args.join(' '));
-            console.warn(...args);
-        }
-    };
     window.API = {
         live: {
             getChannels: async () => {
@@ -269,26 +260,22 @@ test('user analysis click keeps visible channels after Local AI language failure
     window.clearInterval(channelList._epgRefreshTimer);
     window.clearInterval(settingsPage.liveAnalysisPollTimer);
 
-    const notice = window.document.querySelector('.laya-language-notice');
     const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent);
     const languageBadges = [...window.document.querySelectorAll('.variant-language')].map(item => item.textContent);
     const groupNames = [...window.document.querySelectorAll('.group-name')].map(item => item.textContent);
 
     assert.equal(channelLoads, 2);
-    assert.equal(layaBatchCalls, 1);
-    assert.ok(notice);
-    assert.match(notice.textContent, /Local AI language detection failed/);
+    assert.equal(layaBatchCalls, 0);
     assert.deepEqual(afterNames, ['Sports One', 'Sports One']);
     assert.deepEqual([...new Set(languageBadges)].sort(), ['EN 100%', 'FR 100%']);
     assert.deepEqual(groupNames, ['Favorites', 'Sports France']);
     assert.equal(window.document.querySelector('.empty-state'), null);
     assert.equal(window.document.querySelector('#channel-list > .loading'), null);
-    assert.equal(channelList.languageDetectionFallback, true);
+    assert.equal(channelList.languageDetectionFallback, false);
     assert.equal(channelList.logicalChannels[0].variants.length, 2);
-    assert.ok(consoleWarnings.some(line => line.includes('Local AI language detection unavailable')));
 });
 
-test('successful Standard fallback after Local AI failure keeps channels and reports the AI failure', async () => {
+test('hybrid analysis completion relies on persisted projection without Standard browser fallback', async () => {
     const { window } = createDom();
     let rebuildClicked = 0;
     let layaBatchCalls = 0;
@@ -345,16 +332,13 @@ test('successful Standard fallback after Local AI failure keeps channels and rep
     window.clearInterval(settingsPage.liveAnalysisPollTimer);
 
     const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent);
-    const notice = window.document.querySelector('.laya-language-notice');
 
-    assert.equal(layaBatchCalls, 1);
+    assert.equal(layaBatchCalls, 0);
     assert.deepEqual(afterNames, ['Sports One', 'Sports One']);
     assert.equal(window.document.querySelector('.empty-state'), null);
     assert.equal(window.document.querySelector('#channel-list > .loading'), null);
-    assert.ok(notice);
-    assert.match(notice.textContent, /Local AI language detection failed/);
-    assert.match(notice.textContent, /Standard matching/);
-    assert.equal(channelList.languageDetectionFallback, true);
+    assert.equal(window.document.querySelector('.laya-language-notice'), null);
+    assert.equal(channelList.languageDetectionFallback, false);
 });
 
 test('successful Laya language batch updates visible variant languages without Standard fallback', async () => {

test/liveProjectionRoutes.test.js

@@ -15,6 +15,10 @@ test('projection rebuild route requires authentication and admin authorization',
     assert.match(routeSource, /router\.post\('\/projection\/rebuild', requireAuth, requireAdmin,/);
 });
 
+test('projection rebuild route requests a complete analysis', () => {
+    assert.match(routeSource, /liveProjection\.startRebuild\(null, \{ analyzeAll: true \}\)/);
+});
+
 test('projection authorization distinguishes viewer and admin roles', () => {
     assert.equal(isAdmin({ role: 'viewer' }), false);
     assert.equal(isAdmin({ role: 'admin' }), true);

test/t25Frontend.test.js

@@ -25,6 +25,7 @@ function createChannelList() {
     window.eval(script('js/services/channelLanguageDetection.js'));
     window.eval(script('js/services/channelLanguageFusion.js'));
     window.eval(script('js/services/channelVariantGrouping.js'));
+    window.eval(script('js/services/layaStatusPresentation.js'));
     window.eval(script('js/services/liveChannelProjection.js'));
     window.eval(`${script('js/components/ChannelList.js')}\nwindow.ChannelList = ChannelList;`);
     window.app = {
@@ -95,6 +96,36 @@ test('language selection filters the complete loaded catalog immediately without
     dom.window.close();
 });
 
+test('loading the projected channel list does not call Laya language detection during display', async () => {
+    const { dom, window, channelList } = createChannelList();
+    window.app.player.settings.languageDetectionMode = 'laya';
+    let layaStatusCalls = 0;
+    let layaBatchCalls = 0;
+    window.API = {
+        live: { getChannels: async () => projectedResponse() },
+        laya: {
+            getStatus: async () => {
+                layaStatusCalls += 1;
+                return { modelPresent: true, runtimeAvailable: true };
+            },
+            detectLanguages: async () => {
+                layaBatchCalls += 1;
+                return { results: [] };
+            }
+        }
+    };
+
+    try {
+        await channelList.loadAllChannels();
+        await wait(20);
+
+        assert.equal(layaStatusCalls, 0);
+        assert.equal(layaBatchCalls, 0);
+    } finally {
+        dom.window.close();
+    }
+});
+
 test('Escape clears channel search while preserving language-panel Escape behavior', async () => {
     const { dom, window, channelList } = createChannelList();
     window.API = { live: { getChannels: async () => projectedResponse() } };

test/t36LanguagePersistenceRegression.test.js

@@ -0,0 +1,171 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Database = require('better-sqlite3');
+const {
+    ensureColumns,
+    LIVE_PROJECTION_COLUMNS,
+    LIVE_PROJECTION_SCHEMA_SQL
+} = require('../server/db/liveProjectionSchema');
+const liveProjection = require('../server/services/liveProjectionService');
+
+function createDb() {
+    const db = new Database(':memory:');
+    db.exec(`
+        CREATE TABLE categories (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            category_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            parent_id TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE playlist_items (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            category_id TEXT,
+            parent_id TEXT,
+            stream_icon TEXT,
+            stream_url TEXT,
+            container_extension TEXT,
+            rating REAL,
+            year TEXT,
+            added_at TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            is_favorite INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE favorites (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            user_id INTEGER NOT NULL,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            item_type TEXT NOT NULL
+        );
+    `);
+    ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
+    db.exec(LIVE_PROJECTION_SCHEMA_SQL);
+    return db;
+}
+
+function seedChannel(db, id, name, extra = {}) {
+    db.prepare(`
+        INSERT INTO playlist_items
+            (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data,
+             language_code, language_confidence, language_detector, language_input_hash)
+        VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, ?)
+    `).run(
+        `1:${id}`,
+        id,
+        name,
+        `https://streams.test/${id}.m3u8`,
+        extra.languageCode || null,
+        extra.languageConfidence || null,
+        extra.languageDetector || null,
+        extra.languageInputHash || null
+    );
+}
+
+test('background projection rebuild keeps a persisted stream language instead of redetecting it', () => {
+    const db = createDb();
+    seedChannel(db, 'persisted', 'World News', {
+        languageCode: 'fr',
+        languageConfidence: 0.99,
+        languageDetector: 'laya',
+        languageInputHash: 'stable-language-input'
+    });
+
+    try {
+        liveProjection.rebuild(db, { mode: 'algo' });
+        const row = db.prepare(`
+            SELECT language_code, language_detector, language_confidence
+            FROM playlist_items
+            WHERE id = '1:persisted'
+        `).get();
+
+        assert.equal(row.language_code, 'fr');
+        assert.equal(row.language_detector, 'laya');
+        assert.equal(Number(row.language_confidence), 0.99);
+    } finally {
+        db.close();
+    }
+});
+
+test('Laya analysis for one newly detected stream does not send old catalog pairs', async () => {
+    const db = createDb();
+    for (let index = 0; index < 8; index++) {
+        seedChannel(db, `old-${index}`, `beIN Sports 1 legacy ${index}`, {
+            languageCode: 'fr',
+            languageConfidence: 0.95,
+            languageDetector: 'laya',
+            languageInputHash: `old-${index}-hash`
+        });
+    }
+    seedChannel(db, 'new-stream', 'beIN Sports 1 new feed');
+
+    const oldCatalogPairs = [];
+    const layaClient = {
+        identityHintBatch: async pairs => {
+            for (const pair of pairs) {
+                if (!pair.left.name.includes('new feed') && !pair.right.name.includes('new feed')) {
+                    oldCatalogPairs.push(pair);
+                }
+            }
+            return pairs.map(pair => ({
+                id: pair.id,
+                sameIdentityProbability: 0.9,
+                confidence: 0.9,
+                family: 'bein sports',
+                number: '1'
+            }));
+        }
+    };
+
+    try {
+        await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient });
+        assert.equal(oldCatalogPairs.length, 0, `unexpected old-catalog Laya pairs=${oldCatalogPairs.length}`);
+    } finally {
+        db.close();
+    }
+});
+
+test('analyzeAll sends persisted catalog pairs to Laya', async () => {
+    const db = createDb();
+    seedChannel(db, 'persisted-a', 'beIN Sports 1 east', {
+        languageCode: 'fr',
+        languageConfidence: 0.95,
+        languageDetector: 'laya',
+        languageInputHash: 'persisted-a-hash'
+    });
+    seedChannel(db, 'persisted-b', 'beIN Sports 1 west', {
+        languageCode: 'fr',
+        languageConfidence: 0.95,
+        languageDetector: 'laya',
+        languageInputHash: 'persisted-b-hash'
+    });
+
+    let layaPairs = 0;
+    const layaClient = {
+        identityHintBatch: async pairs => {
+            layaPairs += pairs.length;
+            return pairs.map(pair => ({
+                id: pair.id,
+                sameIdentityProbability: 0.9,
+                confidence: 0.9,
+                family: 'bein sports',
+                number: '1'
+            }));
+        }
+    };
+
+    try {
+        await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient, analyzeAll: true });
+        assert.ok(layaPairs > 0);
+    } finally {
+        db.close();
+    }
+});

test/t36SyncIncrementalRegression.test.js

@@ -0,0 +1,201 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Database = require('better-sqlite3');
+
+function createDb() {
+    const db = new Database(':memory:');
+    db.exec(`
+        CREATE TABLE categories (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            category_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            parent_id TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE playlist_items (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            category_id TEXT,
+            parent_id TEXT,
+            stream_icon TEXT,
+            stream_url TEXT,
+            container_extension TEXT,
+            rating REAL,
+            year TEXT,
+            added_at TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            is_favorite INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE sync_status (
+            source_id INTEGER NOT NULL,
+            type TEXT NOT NULL,
+            last_sync INTEGER NOT NULL,
+            status TEXT,
+            error TEXT,
+            PRIMARY KEY (source_id, type)
+        );
+    `);
+    return db;
+}
+
+function seedLive(db, sourceId, streamId, name = `Channel ${streamId}`) {
+    db.prepare(`
+        INSERT INTO playlist_items
+            (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data)
+        VALUES (?, ?, ?, 'live', ?, 'sports', '', ?, '{}')
+    `).run(`${sourceId}:${streamId}`, sourceId, String(streamId), name, `https://streams.test/${streamId}.m3u8`);
+}
+
+function loadSyncServiceWithDb(db) {
+    const sqlite = require('../server/db/sqlite');
+    const originalGetDb = sqlite.getDb;
+    sqlite.getDb = () => db;
+    const syncServicePath = require.resolve('../server/services/syncService');
+    delete require.cache[syncServicePath];
+    const syncService = require('../server/services/syncService');
+    return {
+        syncService,
+        restore() {
+            sqlite.getDb = originalGetDb;
+            delete require.cache[syncServicePath];
+        }
+    };
+}
+
+function stream(streamId, name = `Channel ${streamId}`) {
+    return {
+        stream_id: streamId,
+        name,
+        category_id: 'sports',
+        stream_icon: '',
+        added: 'now'
+    };
+}
+
+function patchLiveProjection() {
+    const liveProjection = require('../server/services/liveProjectionService');
+    const originalQueueRebuild = liveProjection.queueRebuild;
+    const originalStartRebuild = liveProjection.startRebuild;
+    const requests = [];
+    liveProjection.queueRebuild = async (...args) => {
+        requests.push({ method: 'queueRebuild', args });
+        return { status: 'success' };
+    };
+    liveProjection.startRebuild = (...args) => {
+        requests.push({ method: 'startRebuild', args });
+        return { promise: Promise.resolve({ status: 'success' }) };
+    };
+    return {
+        requests,
+        restore() {
+            liveProjection.queueRebuild = originalQueueRebuild;
+            liveProjection.startRebuild = originalStartRebuild;
+        }
+    };
+}
+
+function patchLanguageMode(mode) {
+    const appDb = require('../server/db');
+    const originalGet = appDb.settings.get;
+    appDb.settings.get = async () => ({ ...appDb.getDefaultSettings(), languageDetectionMode: mode });
+    return () => { appDb.settings.get = originalGet; };
+}
+
+test('saveStreams returns only newly inserted live stream ids, not updated ids', async () => {
+    const db = createDb();
+    seedLive(db, 1, 'old', 'Old Sports');
+    const { syncService, restore } = loadSyncServiceWithDb(db);
+
+    try {
+        const result = await syncService.saveStreams(1, 'live', [
+            stream('old', 'Old Sports Updated'),
+            stream('new', 'New Sports')
+        ], { skipPurge: true });
+        const newIds = result?.newIds || new Set();
+        assert.deepEqual([...newIds].sort(), ['1:new']);
+    } finally {
+        restore();
+        db.close();
+    }
+});
+
+test('syncXtream with new live ids requests one targeted analysis for only those ids', async () => {
+    const db = createDb();
+    seedLive(db, 1, 'old', 'Old Sports');
+    const { syncService, restore } = loadSyncServiceWithDb(db);
+    const liveProjectionPatch = patchLiveProjection();
+    const restoreLanguageMode = patchLanguageMode('laya');
+    const xtreamApi = require('../server/services/xtreamApi');
+    const originalCreateFromSource = xtreamApi.createFromSource;
+    xtreamApi.createFromSource = () => ({
+        getLiveCategories: async () => [{ category_id: 'sports', category_name: 'Sports' }],
+        getLiveStreams: async () => [stream('old', 'Old Sports Updated'), stream('new', 'New Sports')],
+        getVodCategories: async () => [],
+        getVodStreams: async () => [],
+        getSeriesCategories: async () => [],
+        getSeries: async () => [],
+        getXmltvUrl: () => { throw new Error('no epg'); }
+    });
+
+    try {
+        await syncService.syncXtream({ id: 1, name: 'Xtream Fixture' });
+        assert.deepEqual(liveProjectionPatch.requests, [{
+            method: 'startRebuild',
+            args: [undefined, { mode: 'laya', scopeItemIds: ['1:new'] }]
+        }]);
+    } finally {
+        xtreamApi.createFromSource = originalCreateFromSource;
+        restoreLanguageMode();
+        liveProjectionPatch.restore();
+        restore();
+        db.close();
+    }
+});
+
+test('syncM3u aggregates new ids from all batches and requests one final targeted analysis', async () => {
+    const db = createDb();
+    seedLive(db, 1, 'old-a', 'Old A');
+    seedLive(db, 1, 'old-b', 'Old B');
+    const { syncService, restore } = loadSyncServiceWithDb(db);
+    const liveProjectionPatch = patchLiveProjection();
+    const restoreLanguageMode = patchLanguageMode('laya');
+    const m3uParser = require('../server/services/m3uParser');
+    const originalFetchAndParseStreaming = m3uParser.fetchAndParseStreaming;
+    m3uParser.fetchAndParseStreaming = async function* () {
+        yield {
+            channels: [
+                { id: 'old-a', name: 'Old A Updated', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/old-a.m3u8' },
+                { id: 'new-a', name: 'New A', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/new-a.m3u8' }
+            ],
+            groups: ['Sports']
+        };
+        yield {
+            channels: [
+                { id: 'old-b', name: 'Old B Updated', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/old-b.m3u8' },
+                { id: 'new-b', name: 'New B', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/new-b.m3u8' }
+            ],
+            groups: ['News']
+        };
+    };
+
+    try {
+        await syncService.syncM3u({ id: 1, name: 'M3U Fixture', url: 'https://playlist.test/live.m3u' });
+        assert.deepEqual(liveProjectionPatch.requests, [{
+            method: 'startRebuild',
+            args: [undefined, { mode: 'laya', scopeItemIds: ['1:new-a', '1:new-b'] }]
+        }]);
+    } finally {
+        m3uParser.fetchAndParseStreaming = originalFetchAndParseStreaming;
+        restoreLanguageMode();
+        liveProjectionPatch.restore();
+        restore();
+        db.close();
+    }
+});