From 4319e64e1e4522910ec989793454db8e20c45228 Mon Sep 17 00:00:00 2001 From: blomios Date: Wed, 7 Oct 2026 23:12:42 +0200 Subject: [PATCH] feat: persist incremental live channel analysis Persist live channel language metadata across background projection rebuilds and target Local AI identity analysis to newly detected streams unless a manual full analysis is requested. Co-Authored-By: Claude Opus 4.8 --- public/js/components/ChannelList.js | 16 +- public/js/pages/Settings.js | 30 +-- server/routes/live.js | 2 +- server/services/liveProjectionService.js | 72 +++++-- server/services/syncService.js | 33 ++- test/liveAnalysisUserFlowDom.test.js | 54 ++--- test/liveProjectionRoutes.test.js | 4 + test/t25Frontend.test.js | 31 +++ test/t36LanguagePersistenceRegression.test.js | 171 +++++++++++++++ test/t36SyncIncrementalRegression.test.js | 201 ++++++++++++++++++ 10 files changed, 516 insertions(+), 98 deletions(-) create mode 100644 test/t36LanguagePersistenceRegression.test.js create mode 100644 test/t36SyncIncrementalRegression.test.js diff --git a/public/js/components/ChannelList.js b/public/js/components/ChannelList.js index 1b541f4..73825e1 100644 --- a/public/js/components/ChannelList.js +++ b/public/js/components/ChannelList.js @@ -195,7 +195,7 @@ class ChannelList { clearTimeout(searchTimeout); searchTimeout = setTimeout(() => { if (this.projectedLogicalChannels || this.channels.length) this.render(); - else this.loadChannels({ refreshLanguages: false }); + else this.loadChannels(); }, this.projectedLogicalChannels || this.channels.length ? 75 : 250); }); @@ -366,13 +366,11 @@ class ChannelList { } setLanguageDetectionMode(mode) { - const safeMode = ['algo', 'laya', 'hybrid'].includes(mode) ? mode : 'algo'; this.layaLanguageResults.clear(); this.languageDetectionNotice = ''; this.languageDetectionFallback = false; if (this.channels.length > 0) { this.render(); - void this.refreshLayaLanguages(safeMode); } } @@ -945,7 +943,7 @@ class ChannelList { /** * Load channels from selected source */ - async loadChannels({ refreshLanguages = true } = {}) { + async loadChannels() { if (this.isLoading) return; this.isLoading = true; this.currentRenderId = null; // Reset render tracking @@ -962,7 +960,7 @@ class ChannelList { if (!sourceValue) { // Load from all sources - await this.loadAllChannels({ refreshLanguages }); + await this.loadAllChannels(); this.isLoading = false; return; } @@ -974,7 +972,6 @@ class ChannelList { if (await this.loadProjectedChannels(parseInt(id, 10))) { this.render(); - if (refreshLanguages) void this.refreshLayaLanguages(); return; } @@ -991,7 +988,6 @@ class ChannelList { ]); this.render(); - if (refreshLanguages) void this.refreshLayaLanguages(); } catch (err) { console.error('Error loading channels:', err); if (previousState.channels.length > 0) { @@ -1011,7 +1007,7 @@ class ChannelList { /** * Load channels from all enabled sources */ - async loadAllChannels({ refreshLanguages = true } = {}) { + async loadAllChannels() { const previousState = { channels: this.channels, groups: this.groups, @@ -1026,7 +1022,6 @@ class ChannelList { if (await this.loadProjectedChannels()) { this.render(); - if (refreshLanguages) void this.refreshLayaLanguages(); return; } @@ -1047,7 +1042,6 @@ class ChannelList { this.loadFavorites() ]); this.render(); - if (refreshLanguages) void this.refreshLayaLanguages(); } catch (err) { console.error('Error loading all channels:', err); if (previousState.channels.length > 0) { @@ -1124,7 +1118,7 @@ class ChannelList { } catch {} this.renderLanguageFilter(); if (this.projectedLogicalChannels || this.channels.length) this.render(); - else this.loadChannels({ refreshLanguages: false }); + else this.loadChannels(); } closeLanguagePanel() { diff --git a/public/js/pages/Settings.js b/public/js/pages/Settings.js index 9dfba4d..6fd5539 100644 --- a/public/js/pages/Settings.js +++ b/public/js/pages/Settings.js @@ -197,30 +197,7 @@ class SettingsPage { clearInterval(this.liveAnalysisPollTimer); this.liveAnalysisPollTimer = null; if (presentation.complete) { - const mode = this.app.player?.settings?.languageDetectionMode || 'algo'; - await this.app.channelList?.loadChannels({ - refreshLanguages: mode !== 'laya' && mode !== 'hybrid' - }); - if (mode === 'laya' || mode === 'hybrid') { - this.liveAnalysisStatus = { - status: 'building', - phase: 'laya_language', - languageProcessed: null, - languageTotal: null - }; - this.renderLiveAnalysisStatus(this.liveAnalysisStatus); - await this.app.channelList?.refreshLayaLanguages(mode, { - onProgress: ({ processed, total }) => { - this.liveAnalysisStatus = { - status: 'building', - phase: 'laya_language', - languageProcessed: processed, - languageTotal: total - }; - this.renderLiveAnalysisStatus(this.liveAnalysisStatus); - } - }); - } + await this.app.channelList?.loadChannels(); this.liveAnalysisStatus = status; this.renderLiveAnalysisStatus(status); } @@ -272,11 +249,6 @@ class SettingsPage { this.layaStatus = { runtimeAvailable: false, modelPresent: false, error: err.message }; } this.renderLayaStatus(this.layaStatus); - const mode = this.app.player?.settings?.languageDetectionMode || 'algo'; - if (mode !== 'algo' && this.layaStatus?.modelPresent && this.layaStatus?.runtimeAvailable && - this.app.channelList?.languageDetectionNotice) { - void this.app.channelList.refreshLayaLanguages(mode); - } if (this.layaStatus?.downloading) this.startLayaStatusPolling(); return this.layaStatus; } diff --git a/server/routes/live.js b/server/routes/live.js index 20b1629..e0400cb 100644 --- a/server/routes/live.js +++ b/server/routes/live.js @@ -71,7 +71,7 @@ router.get('/projection/status', requireAuth, (req, res) => { router.post('/projection/rebuild', requireAuth, requireAdmin, (req, res) => { try { - const started = liveProjection.startRebuild(); + const started = liveProjection.startRebuild(null, { analyzeAll: true }); started.promise.catch(error => { console.error('[LiveProjection] Manual rebuild failed:', error.message); }); diff --git a/server/services/liveProjectionService.js b/server/services/liveProjectionService.js index bd87583..7e85b01 100644 --- a/server/services/liveProjectionService.js +++ b/server/services/liveProjectionService.js @@ -69,6 +69,22 @@ function stableHash(value) { return crypto.createHash('sha256').update(value).digest('hex'); } +function languageInputHash(item) { + return stableHash(`${item.name || ''}\0${item.category_name || ''}\0${item.tvg_id || ''}`); +} + +function hasPersistedLanguage(item) { + if (!/^[a-z]{2,3}$/i.test(item.language_code || '') || String(item.language_code).toLowerCase() === 'und') return false; + if (!item.language_detector || !item.language_input_hash) return false; + const persistedHash = String(item.language_input_hash); + return !/^[a-f0-9]{64}$/i.test(persistedHash) || persistedHash === languageInputHash(item); +} + +function parseNullableData(value) { + if (value == null) return null; + return parseData(value); +} + function projectCatalog(items, options = {}) { const candidateBuckets = new Map(); for (const item of items) { @@ -82,11 +98,22 @@ function projectCatalog(items, options = {}) { qualityLabel: quality.label, qualityRank: quality.rank }; - const language = detectLanguage({ ...projected, data: parseData(projected.data) }); - projected.languageCode = language.code; - projected.languageConfidence = language.confidence; - projected.languageScores = language.scores; - projected.languageEvidence = language.evidence; + if (hasPersistedLanguage(item)) { + projected.languageCode = item.language_code; + projected.languageConfidence = item.language_confidence; + projected.languageScores = parseNullableData(item.language_scores); + projected.languageEvidence = parseNullableData(item.language_evidence); + projected.languageDetector = item.language_detector; + projected.languageInputHash = item.language_input_hash; + } else { + const language = detectLanguage({ ...projected, data: parseData(projected.data) }); + projected.languageCode = language.code; + projected.languageConfidence = language.confidence; + projected.languageScores = language.scores; + projected.languageEvidence = language.evidence; + projected.languageDetector = 'algorithm'; + projected.languageInputHash = languageInputHash(projected); + } const itemIdentity = identity(fingerprint); const candidateKey = itemIdentity.family && itemIdentity.number ? `${itemIdentity.family}:${itemIdentity.number}` @@ -124,7 +151,14 @@ function safeIdentityDescriptor(item) { }; } -function identityCandidatePairs(items) { +function identityCandidatePairs(items, options = {}) { + const hasScope = Object.hasOwn(options, 'scopeItemIds'); + const scopeItemIds = hasScope + ? new Set(Array.from(options.scopeItemIds || [], String)) + : null; + const isScoped = item => !scopeItemIds + || scopeItemIds.has(String(item.id)) + || scopeItemIds.has(String(item.item_id)); const buckets = new Map(); for (const item of items) { const fingerprint = normalizeChannelName(item.name); @@ -138,6 +172,7 @@ function identityCandidatePairs(items) { for (const bucket of buckets.values()) { for (let left = 0; left < bucket.length; left++) { for (let right = left + 1; right < bucket.length; right++) { + if (!isScoped(bucket[left]) && !isScoped(bucket[right])) continue; if (bucket[left].fingerprint === bucket[right].fingerprint) continue; if (scoreCandidate(bucket[left], bucket[right], { mode: 'algo' }).blocker) continue; pairs.push({ @@ -154,7 +189,10 @@ function identityCandidatePairs(items) { async function collectIdentityHints(items, mode, layaClient, options = {}) { if (mode === 'algo' || !layaClient) return new Map(); - const pairs = identityCandidatePairs(items); + const pairOptions = Object.hasOwn(options, 'scopeItemIds') + ? { scopeItemIds: options.scopeItemIds } + : {}; + const pairs = identityCandidatePairs(items, pairOptions); const total = Math.ceil(pairs.length / 64); let processed = 0; const hints = new Map(); @@ -227,7 +265,8 @@ function markProjectionBuilding(database = null) { function loadProjectionItems(database) { return database.prepare(` SELECT p.id, p.source_id, p.item_id, p.name, p.category_id, p.data, - c.name AS category_name + p.language_code, p.language_confidence, p.language_scores, p.language_evidence, + p.language_detector, p.language_input_hash, c.name AS category_name FROM playlist_items p LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id WHERE p.type = 'live' @@ -252,9 +291,7 @@ function rebuild(database = null, options = {}) { database.prepare('DELETE FROM logical_channels').run(); database.prepare(` UPDATE playlist_items SET logical_channel_id = NULL, grouping_fingerprint = NULL, - grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL, - language_code = NULL, language_confidence = NULL, language_scores = NULL, - language_evidence = NULL, language_detector = NULL, language_input_hash = NULL + grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL WHERE type = 'live' `).run(); const insertLogical = database.prepare(` @@ -266,7 +303,7 @@ function rebuild(database = null, options = {}) { UPDATE playlist_items SET logical_channel_id = ?, grouping_fingerprint = ?, grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?, language_code = ?, language_confidence = ?, language_scores = ?, - language_evidence = ?, language_detector = 'algorithm', language_input_hash = ? + language_evidence = ?, language_detector = ?, language_input_hash = ? WHERE id = ? AND type = 'live' `); const now = Date.now(); @@ -276,8 +313,9 @@ function rebuild(database = null, options = {}) { updateVariant.run(group.id, group.fingerprint, variant.groupingInputHash, variant.qualityLabel, variant.qualityRank, variant.variantRank, variant.languageCode, variant.languageConfidence, - JSON.stringify(variant.languageScores), JSON.stringify(variant.languageEvidence), - stableHash(`${variant.name || ''}\0${variant.category_name || ''}\0${variant.tvg_id || ''}`), variant.id); + variant.languageScores == null ? null : JSON.stringify(variant.languageScores), + variant.languageEvidence == null ? null : JSON.stringify(variant.languageEvidence), + variant.languageDetector, variant.languageInputHash, variant.id); } } database.prepare(` @@ -307,7 +345,13 @@ async function rebuildWithSignals(database = null, options = {}) { let identityHints = new Map(); if (mode !== 'algo') { const layaClient = options.layaClient || new (require('./layaService').LayaService)(); + const identityOptions = options.analyzeAll === true + ? {} + : Object.hasOwn(options, 'scopeItemIds') + ? { scopeItemIds: options.scopeItemIds } + : { scopeItemIds: items.filter(item => !hasPersistedLanguage(item)).map(item => item.id) }; identityHints = await collectOptionalIdentityHints(items, mode, layaClient, { + ...identityOptions, onProgress: ({ processed, total }) => updateProjectionProgress(database, 'laya_identity', { layaProcessed: processed, layaTotal: total diff --git a/server/services/syncService.js b/server/services/syncService.js index 94c36f3..b7efe6e 100644 --- a/server/services/syncService.js +++ b/server/services/syncService.js @@ -8,6 +8,18 @@ const liveProjection = require('./liveProjectionService'); // Sync tracking const activeSyncs = new Set(); // sourceId +async function rebuildLiveProjection(newIds) { + const currentSettings = await settings.get(); + const mode = currentSettings.languageDetectionMode; + if (newIds.size > 0 && (mode === 'laya' || mode === 'hybrid')) { + return liveProjection.startRebuild(undefined, { + mode, + scopeItemIds: [...newIds] + }).promise; + } + return liveProjection.queueRebuild(); +} + class SyncService { constructor() { this.lastSyncTime = null; // Track when global sync last completed @@ -183,8 +195,8 @@ class SyncService { // 2. Live Streams console.log(`[Sync] Fetching Live Streams for ${source.name}`); const liveStreams = await api.getLiveStreams(); - await this.saveStreams(source.id, 'live', liveStreams); - await liveProjection.queueRebuild(); + const { newIds } = await this.saveStreams(source.id, 'live', liveStreams); + await rebuildLiveProjection(newIds); // 3. VOD Categories console.log(`[Sync] Fetching VOD Categories for ${source.name}`); @@ -259,15 +271,17 @@ class SyncService { * @param {string} type - Type of items (live, movie, series) * @param {Array} items - Items to save * @param {Object} options - Options { skipPurge: boolean } - * @returns {Set} Set of synced IDs (for external purge if skipPurge was true) + * @returns {{ syncedIds: Set, newIds: Set }} Synced and newly inserted IDs */ async saveStreams(sourceId, type, items, options = {}) { - if (!items || items.length === 0) return new Set(); + if (!items || items.length === 0) return { syncedIds: new Set(), newIds: new Set() }; const db = getDb(); const { skipPurge = false } = options; // Collect all IDs we're syncing const syncedIds = new Set(); + const newIds = new Set(); + const existingItem = db.prepare('SELECT 1 FROM playlist_items WHERE id = ?'); const stmt = db.prepare(` INSERT INTO playlist_items ( @@ -316,6 +330,7 @@ class SyncService { const id = `${sourceId}:${itemId}`; syncedIds.add(id); + if (!existingItem.get(id)) newIds.add(id); stmt.run( id, @@ -349,7 +364,7 @@ class SyncService { } console.log(`[Sync] Saved ${items.length} ${type} items`); - return syncedIds; + return { syncedIds, newIds }; } /** @@ -511,6 +526,7 @@ class SyncService { const allGroups = new Set(); const allSyncedIds = new Set(); // Collect IDs across all batches + const allNewIds = new Set(); let totalChannels = 0; let batchCount = 0; @@ -530,8 +546,9 @@ class SyncService { // Save this batch immediately (skip purge - we'll do it at the end) if (playlistItems.length > 0) { - const batchIds = await this.saveStreams(source.id, 'live', playlistItems, { skipPurge: true }); - batchIds.forEach(id => allSyncedIds.add(id)); + const { syncedIds, newIds } = await this.saveStreams(source.id, 'live', playlistItems, { skipPurge: true }); + syncedIds.forEach(id => allSyncedIds.add(id)); + newIds.forEach(id => allNewIds.add(id)); totalChannels += playlistItems.length; } @@ -561,7 +578,7 @@ class SyncService { })); await this.saveCategories(source.id, 'live', categories); - await liveProjection.queueRebuild(); + await rebuildLiveProjection(allNewIds); console.log(`[Sync] M3U sync complete for ${source.name}`); } diff --git a/test/liveAnalysisUserFlowDom.test.js b/test/liveAnalysisUserFlowDom.test.js index 1afcc5e..50b9af5 100644 --- a/test/liveAnalysisUserFlowDom.test.js +++ b/test/liveAnalysisUserFlowDom.test.js @@ -141,12 +141,12 @@ test('Settings renders Laya batch progress and removes misleading values when it assert.equal(bar.style.width, ''); }); -test('analysis remains visibly busy while Laya language batches run after projection completes', async () => { +test('analysis completion reloads the persisted projection', async () => { const { window } = createDom(); let rebuildClicked = false; - let languageRefreshStarted = false; - let releaseLanguageRefresh; - const languageRefresh = new Promise(resolve => { releaseLanguageRefresh = resolve; }); + let projectionReloadStarted = false; + let releaseProjectionReload; + const projectionReload = new Promise(resolve => { releaseProjectionReload = resolve; }); window.API = { live: { getProjectionStatus: async () => rebuildClicked @@ -176,11 +176,11 @@ test('analysis remains visibly busy while Laya language batches run after projec saveSettings() {} }, channelList: { - loadChannels: async () => {}, - refreshLayaLanguages: async () => { - languageRefreshStarted = true; - await languageRefresh; - } + loadChannels: async () => { + projectionReloadStarted = true; + await projectionReload; + }, + refreshLayaLanguages: async () => assert.fail('Browser-side Laya refresh must not run') } }; window.app = app; @@ -193,28 +193,19 @@ test('analysis remains visibly busy while Laya language batches run after projec window.document.getElementById('live-analysis-button').click(); try { - await waitFor(() => languageRefreshStarted, 'Expected Laya language refresh to start after projection completion'); - assert.notEqual(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete'); - assert.equal(window.document.getElementById('live-analysis-progress').hidden, false); + await waitFor(() => projectionReloadStarted, 'Expected persisted projection reload after analysis completion'); + assert.equal(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete'); } finally { - releaseLanguageRefresh(); + releaseProjectionReload(); window.clearInterval(settingsPage.liveAnalysisPollTimer); } }); -test('user analysis click keeps visible channels after Local AI language failure', async () => { +test('user analysis click reloads persisted languages without a browser-side Laya batch', async () => { const { window } = createDom(); let rebuildClicked = 0; let layaBatchCalls = 0; let channelLoads = 0; - const consoleWarnings = []; - window.console = { - ...console, - warn: (...args) => { - consoleWarnings.push(args.join(' ')); - console.warn(...args); - } - }; window.API = { live: { getChannels: async () => { @@ -269,26 +260,22 @@ test('user analysis click keeps visible channels after Local AI language failure window.clearInterval(channelList._epgRefreshTimer); window.clearInterval(settingsPage.liveAnalysisPollTimer); - const notice = window.document.querySelector('.laya-language-notice'); const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent); const languageBadges = [...window.document.querySelectorAll('.variant-language')].map(item => item.textContent); const groupNames = [...window.document.querySelectorAll('.group-name')].map(item => item.textContent); assert.equal(channelLoads, 2); - assert.equal(layaBatchCalls, 1); - assert.ok(notice); - assert.match(notice.textContent, /Local AI language detection failed/); + assert.equal(layaBatchCalls, 0); assert.deepEqual(afterNames, ['Sports One', 'Sports One']); assert.deepEqual([...new Set(languageBadges)].sort(), ['EN 100%', 'FR 100%']); assert.deepEqual(groupNames, ['Favorites', 'Sports France']); assert.equal(window.document.querySelector('.empty-state'), null); assert.equal(window.document.querySelector('#channel-list > .loading'), null); - assert.equal(channelList.languageDetectionFallback, true); + assert.equal(channelList.languageDetectionFallback, false); assert.equal(channelList.logicalChannels[0].variants.length, 2); - assert.ok(consoleWarnings.some(line => line.includes('Local AI language detection unavailable'))); }); -test('successful Standard fallback after Local AI failure keeps channels and reports the AI failure', async () => { +test('hybrid analysis completion relies on persisted projection without Standard browser fallback', async () => { const { window } = createDom(); let rebuildClicked = 0; let layaBatchCalls = 0; @@ -345,16 +332,13 @@ test('successful Standard fallback after Local AI failure keeps channels and rep window.clearInterval(settingsPage.liveAnalysisPollTimer); const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent); - const notice = window.document.querySelector('.laya-language-notice'); - assert.equal(layaBatchCalls, 1); + assert.equal(layaBatchCalls, 0); assert.deepEqual(afterNames, ['Sports One', 'Sports One']); assert.equal(window.document.querySelector('.empty-state'), null); assert.equal(window.document.querySelector('#channel-list > .loading'), null); - assert.ok(notice); - assert.match(notice.textContent, /Local AI language detection failed/); - assert.match(notice.textContent, /Standard matching/); - assert.equal(channelList.languageDetectionFallback, true); + assert.equal(window.document.querySelector('.laya-language-notice'), null); + assert.equal(channelList.languageDetectionFallback, false); }); test('successful Laya language batch updates visible variant languages without Standard fallback', async () => { diff --git a/test/liveProjectionRoutes.test.js b/test/liveProjectionRoutes.test.js index ae117d4..6eda485 100644 --- a/test/liveProjectionRoutes.test.js +++ b/test/liveProjectionRoutes.test.js @@ -15,6 +15,10 @@ test('projection rebuild route requires authentication and admin authorization', assert.match(routeSource, /router\.post\('\/projection\/rebuild', requireAuth, requireAdmin,/); }); +test('projection rebuild route requests a complete analysis', () => { + assert.match(routeSource, /liveProjection\.startRebuild\(null, \{ analyzeAll: true \}\)/); +}); + test('projection authorization distinguishes viewer and admin roles', () => { assert.equal(isAdmin({ role: 'viewer' }), false); assert.equal(isAdmin({ role: 'admin' }), true); diff --git a/test/t25Frontend.test.js b/test/t25Frontend.test.js index 7dcf81c..5274597 100644 --- a/test/t25Frontend.test.js +++ b/test/t25Frontend.test.js @@ -25,6 +25,7 @@ function createChannelList() { window.eval(script('js/services/channelLanguageDetection.js')); window.eval(script('js/services/channelLanguageFusion.js')); window.eval(script('js/services/channelVariantGrouping.js')); + window.eval(script('js/services/layaStatusPresentation.js')); window.eval(script('js/services/liveChannelProjection.js')); window.eval(`${script('js/components/ChannelList.js')}\nwindow.ChannelList = ChannelList;`); window.app = { @@ -95,6 +96,36 @@ test('language selection filters the complete loaded catalog immediately without dom.window.close(); }); +test('loading the projected channel list does not call Laya language detection during display', async () => { + const { dom, window, channelList } = createChannelList(); + window.app.player.settings.languageDetectionMode = 'laya'; + let layaStatusCalls = 0; + let layaBatchCalls = 0; + window.API = { + live: { getChannels: async () => projectedResponse() }, + laya: { + getStatus: async () => { + layaStatusCalls += 1; + return { modelPresent: true, runtimeAvailable: true }; + }, + detectLanguages: async () => { + layaBatchCalls += 1; + return { results: [] }; + } + } + }; + + try { + await channelList.loadAllChannels(); + await wait(20); + + assert.equal(layaStatusCalls, 0); + assert.equal(layaBatchCalls, 0); + } finally { + dom.window.close(); + } +}); + test('Escape clears channel search while preserving language-panel Escape behavior', async () => { const { dom, window, channelList } = createChannelList(); window.API = { live: { getChannels: async () => projectedResponse() } }; diff --git a/test/t36LanguagePersistenceRegression.test.js b/test/t36LanguagePersistenceRegression.test.js new file mode 100644 index 0000000..9f50d08 --- /dev/null +++ b/test/t36LanguagePersistenceRegression.test.js @@ -0,0 +1,171 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const Database = require('better-sqlite3'); +const { + ensureColumns, + LIVE_PROJECTION_COLUMNS, + LIVE_PROJECTION_SCHEMA_SQL +} = require('../server/db/liveProjectionSchema'); +const liveProjection = require('../server/services/liveProjectionService'); + +function createDb() { + const db = new Database(':memory:'); + db.exec(` + CREATE TABLE categories ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + category_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + parent_id TEXT, + is_hidden INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE playlist_items ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + category_id TEXT, + parent_id TEXT, + stream_icon TEXT, + stream_url TEXT, + container_extension TEXT, + rating REAL, + year TEXT, + added_at TEXT, + is_hidden INTEGER DEFAULT 0, + is_favorite INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE favorites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + item_type TEXT NOT NULL + ); + `); + ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS); + db.exec(LIVE_PROJECTION_SCHEMA_SQL); + return db; +} + +function seedChannel(db, id, name, extra = {}) { + db.prepare(` + INSERT INTO playlist_items + (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data, + language_code, language_confidence, language_detector, language_input_hash) + VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, ?) + `).run( + `1:${id}`, + id, + name, + `https://streams.test/${id}.m3u8`, + extra.languageCode || null, + extra.languageConfidence || null, + extra.languageDetector || null, + extra.languageInputHash || null + ); +} + +test('background projection rebuild keeps a persisted stream language instead of redetecting it', () => { + const db = createDb(); + seedChannel(db, 'persisted', 'World News', { + languageCode: 'fr', + languageConfidence: 0.99, + languageDetector: 'laya', + languageInputHash: 'stable-language-input' + }); + + try { + liveProjection.rebuild(db, { mode: 'algo' }); + const row = db.prepare(` + SELECT language_code, language_detector, language_confidence + FROM playlist_items + WHERE id = '1:persisted' + `).get(); + + assert.equal(row.language_code, 'fr'); + assert.equal(row.language_detector, 'laya'); + assert.equal(Number(row.language_confidence), 0.99); + } finally { + db.close(); + } +}); + +test('Laya analysis for one newly detected stream does not send old catalog pairs', async () => { + const db = createDb(); + for (let index = 0; index < 8; index++) { + seedChannel(db, `old-${index}`, `beIN Sports 1 legacy ${index}`, { + languageCode: 'fr', + languageConfidence: 0.95, + languageDetector: 'laya', + languageInputHash: `old-${index}-hash` + }); + } + seedChannel(db, 'new-stream', 'beIN Sports 1 new feed'); + + const oldCatalogPairs = []; + const layaClient = { + identityHintBatch: async pairs => { + for (const pair of pairs) { + if (!pair.left.name.includes('new feed') && !pair.right.name.includes('new feed')) { + oldCatalogPairs.push(pair); + } + } + return pairs.map(pair => ({ + id: pair.id, + sameIdentityProbability: 0.9, + confidence: 0.9, + family: 'bein sports', + number: '1' + })); + } + }; + + try { + await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient }); + assert.equal(oldCatalogPairs.length, 0, `unexpected old-catalog Laya pairs=${oldCatalogPairs.length}`); + } finally { + db.close(); + } +}); + +test('analyzeAll sends persisted catalog pairs to Laya', async () => { + const db = createDb(); + seedChannel(db, 'persisted-a', 'beIN Sports 1 east', { + languageCode: 'fr', + languageConfidence: 0.95, + languageDetector: 'laya', + languageInputHash: 'persisted-a-hash' + }); + seedChannel(db, 'persisted-b', 'beIN Sports 1 west', { + languageCode: 'fr', + languageConfidence: 0.95, + languageDetector: 'laya', + languageInputHash: 'persisted-b-hash' + }); + + let layaPairs = 0; + const layaClient = { + identityHintBatch: async pairs => { + layaPairs += pairs.length; + return pairs.map(pair => ({ + id: pair.id, + sameIdentityProbability: 0.9, + confidence: 0.9, + family: 'bein sports', + number: '1' + })); + } + }; + + try { + await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient, analyzeAll: true }); + assert.ok(layaPairs > 0); + } finally { + db.close(); + } +}); diff --git a/test/t36SyncIncrementalRegression.test.js b/test/t36SyncIncrementalRegression.test.js new file mode 100644 index 0000000..c4bcab3 --- /dev/null +++ b/test/t36SyncIncrementalRegression.test.js @@ -0,0 +1,201 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const Database = require('better-sqlite3'); + +function createDb() { + const db = new Database(':memory:'); + db.exec(` + CREATE TABLE categories ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + category_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + parent_id TEXT, + is_hidden INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE playlist_items ( + id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL, + item_id TEXT NOT NULL, + type TEXT NOT NULL, + name TEXT NOT NULL, + category_id TEXT, + parent_id TEXT, + stream_icon TEXT, + stream_url TEXT, + container_extension TEXT, + rating REAL, + year TEXT, + added_at TEXT, + is_hidden INTEGER DEFAULT 0, + is_favorite INTEGER DEFAULT 0, + data JSON + ); + CREATE TABLE sync_status ( + source_id INTEGER NOT NULL, + type TEXT NOT NULL, + last_sync INTEGER NOT NULL, + status TEXT, + error TEXT, + PRIMARY KEY (source_id, type) + ); + `); + return db; +} + +function seedLive(db, sourceId, streamId, name = `Channel ${streamId}`) { + db.prepare(` + INSERT INTO playlist_items + (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data) + VALUES (?, ?, ?, 'live', ?, 'sports', '', ?, '{}') + `).run(`${sourceId}:${streamId}`, sourceId, String(streamId), name, `https://streams.test/${streamId}.m3u8`); +} + +function loadSyncServiceWithDb(db) { + const sqlite = require('../server/db/sqlite'); + const originalGetDb = sqlite.getDb; + sqlite.getDb = () => db; + const syncServicePath = require.resolve('../server/services/syncService'); + delete require.cache[syncServicePath]; + const syncService = require('../server/services/syncService'); + return { + syncService, + restore() { + sqlite.getDb = originalGetDb; + delete require.cache[syncServicePath]; + } + }; +} + +function stream(streamId, name = `Channel ${streamId}`) { + return { + stream_id: streamId, + name, + category_id: 'sports', + stream_icon: '', + added: 'now' + }; +} + +function patchLiveProjection() { + const liveProjection = require('../server/services/liveProjectionService'); + const originalQueueRebuild = liveProjection.queueRebuild; + const originalStartRebuild = liveProjection.startRebuild; + const requests = []; + liveProjection.queueRebuild = async (...args) => { + requests.push({ method: 'queueRebuild', args }); + return { status: 'success' }; + }; + liveProjection.startRebuild = (...args) => { + requests.push({ method: 'startRebuild', args }); + return { promise: Promise.resolve({ status: 'success' }) }; + }; + return { + requests, + restore() { + liveProjection.queueRebuild = originalQueueRebuild; + liveProjection.startRebuild = originalStartRebuild; + } + }; +} + +function patchLanguageMode(mode) { + const appDb = require('../server/db'); + const originalGet = appDb.settings.get; + appDb.settings.get = async () => ({ ...appDb.getDefaultSettings(), languageDetectionMode: mode }); + return () => { appDb.settings.get = originalGet; }; +} + +test('saveStreams returns only newly inserted live stream ids, not updated ids', async () => { + const db = createDb(); + seedLive(db, 1, 'old', 'Old Sports'); + const { syncService, restore } = loadSyncServiceWithDb(db); + + try { + const result = await syncService.saveStreams(1, 'live', [ + stream('old', 'Old Sports Updated'), + stream('new', 'New Sports') + ], { skipPurge: true }); + const newIds = result?.newIds || new Set(); + assert.deepEqual([...newIds].sort(), ['1:new']); + } finally { + restore(); + db.close(); + } +}); + +test('syncXtream with new live ids requests one targeted analysis for only those ids', async () => { + const db = createDb(); + seedLive(db, 1, 'old', 'Old Sports'); + const { syncService, restore } = loadSyncServiceWithDb(db); + const liveProjectionPatch = patchLiveProjection(); + const restoreLanguageMode = patchLanguageMode('laya'); + const xtreamApi = require('../server/services/xtreamApi'); + const originalCreateFromSource = xtreamApi.createFromSource; + xtreamApi.createFromSource = () => ({ + getLiveCategories: async () => [{ category_id: 'sports', category_name: 'Sports' }], + getLiveStreams: async () => [stream('old', 'Old Sports Updated'), stream('new', 'New Sports')], + getVodCategories: async () => [], + getVodStreams: async () => [], + getSeriesCategories: async () => [], + getSeries: async () => [], + getXmltvUrl: () => { throw new Error('no epg'); } + }); + + try { + await syncService.syncXtream({ id: 1, name: 'Xtream Fixture' }); + assert.deepEqual(liveProjectionPatch.requests, [{ + method: 'startRebuild', + args: [undefined, { mode: 'laya', scopeItemIds: ['1:new'] }] + }]); + } finally { + xtreamApi.createFromSource = originalCreateFromSource; + restoreLanguageMode(); + liveProjectionPatch.restore(); + restore(); + db.close(); + } +}); + +test('syncM3u aggregates new ids from all batches and requests one final targeted analysis', async () => { + const db = createDb(); + seedLive(db, 1, 'old-a', 'Old A'); + seedLive(db, 1, 'old-b', 'Old B'); + const { syncService, restore } = loadSyncServiceWithDb(db); + const liveProjectionPatch = patchLiveProjection(); + const restoreLanguageMode = patchLanguageMode('laya'); + const m3uParser = require('../server/services/m3uParser'); + const originalFetchAndParseStreaming = m3uParser.fetchAndParseStreaming; + m3uParser.fetchAndParseStreaming = async function* () { + yield { + channels: [ + { id: 'old-a', name: 'Old A Updated', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/old-a.m3u8' }, + { id: 'new-a', name: 'New A', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/new-a.m3u8' } + ], + groups: ['Sports'] + }; + yield { + channels: [ + { id: 'old-b', name: 'Old B Updated', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/old-b.m3u8' }, + { id: 'new-b', name: 'New B', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/new-b.m3u8' } + ], + groups: ['News'] + }; + }; + + try { + await syncService.syncM3u({ id: 1, name: 'M3U Fixture', url: 'https://playlist.test/live.m3u' }); + assert.deepEqual(liveProjectionPatch.requests, [{ + method: 'startRebuild', + args: [undefined, { mode: 'laya', scopeItemIds: ['1:new-a', '1:new-b'] }] + }]); + } finally { + m3uParser.fetchAndParseStreaming = originalFetchAndParseStreaming; + restoreLanguageMode(); + liveProjectionPatch.restore(); + restore(); + db.close(); + } +});