blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: persist incremental live channel analysis

Browse Source 
Persist live channel language metadata across background projection rebuilds and target Local AI identity analysis to newly detected streams unless a manual full analysis is requested.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-07 23:12:42 +02:00
parent 9967fd7adf
commit 4319e64e1e
10 changed files with 516 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
public/js/components/ChannelList.js
View File

@ -195,7 +195,7 @@ class ChannelList {
clearTimeout(searchTimeout);
searchTimeout = setTimeout(() => {
if (this.projectedLogicalChannels || this.channels.length) this.render();
else this.loadChannels({ refreshLanguages: false });
else this.loadChannels();
}, this.projectedLogicalChannels || this.channels.length ? 75 : 250);
});
@ -366,13 +366,11 @@ class ChannelList {
}
setLanguageDetectionMode(mode) {
const safeMode = ['algo', 'laya', 'hybrid'].includes(mode) ? mode : 'algo';
this.layaLanguageResults.clear();
this.languageDetectionNotice = '';
this.languageDetectionFallback = false;
if (this.channels.length > 0) {
this.render();
void this.refreshLayaLanguages(safeMode);
}
}
@ -945,7 +943,7 @@ class ChannelList {
/**
* Load channels from selected source
*/
async loadChannels({ refreshLanguages = true } = {}) {
async loadChannels() {
if (this.isLoading) return;
this.isLoading = true;
this.currentRenderId = null; // Reset render tracking
@ -962,7 +960,7 @@ class ChannelList {
if (!sourceValue) {
// Load from all sources
await this.loadAllChannels({ refreshLanguages });
await this.loadAllChannels();
this.isLoading = false;
return;
}
@ -974,7 +972,6 @@ class ChannelList {
if (await this.loadProjectedChannels(parseInt(id, 10))) {
this.render();
if (refreshLanguages) void this.refreshLayaLanguages();
return;
}
@ -991,7 +988,6 @@ class ChannelList {
]);
this.render();
if (refreshLanguages) void this.refreshLayaLanguages();
} catch (err) {
console.error('Error loading channels:', err);
if (previousState.channels.length > 0) {
@ -1011,7 +1007,7 @@ class ChannelList {
/**
* Load channels from all enabled sources
*/
async loadAllChannels({ refreshLanguages = true } = {}) {
async loadAllChannels() {
const previousState = {
channels: this.channels,
groups: this.groups,
@ -1026,7 +1022,6 @@ class ChannelList {
if (await this.loadProjectedChannels()) {
this.render();
if (refreshLanguages) void this.refreshLayaLanguages();
return;
}
@ -1047,7 +1042,6 @@ class ChannelList {
this.loadFavorites()
]);
this.render();
if (refreshLanguages) void this.refreshLayaLanguages();
} catch (err) {
console.error('Error loading all channels:', err);
if (previousState.channels.length > 0) {
@ -1124,7 +1118,7 @@ class ChannelList {
} catch {}
this.renderLanguageFilter();
if (this.projectedLogicalChannels || this.channels.length) this.render();
else this.loadChannels({ refreshLanguages: false });
else this.loadChannels();
}
closeLanguagePanel() {

30
public/js/pages/Settings.js
View File

@ -197,30 +197,7 @@ class SettingsPage {
clearInterval(this.liveAnalysisPollTimer);
this.liveAnalysisPollTimer = null;
if (presentation.complete) {
const mode = this.app.player?.settings?.languageDetectionMode || 'algo';
await this.app.channelList?.loadChannels({
refreshLanguages: mode !== 'laya' && mode !== 'hybrid'
});
if (mode === 'laya' || mode === 'hybrid') {
this.liveAnalysisStatus = {
status: 'building',
phase: 'laya_language',
languageProcessed: null,
languageTotal: null
};
this.renderLiveAnalysisStatus(this.liveAnalysisStatus);
await this.app.channelList?.refreshLayaLanguages(mode, {
onProgress: ({ processed, total }) => {
this.liveAnalysisStatus = {
status: 'building',
phase: 'laya_language',
languageProcessed: processed,
languageTotal: total
};
this.renderLiveAnalysisStatus(this.liveAnalysisStatus);
}
});
}
await this.app.channelList?.loadChannels();
this.liveAnalysisStatus = status;
this.renderLiveAnalysisStatus(status);
}
@ -272,11 +249,6 @@ class SettingsPage {
this.layaStatus = { runtimeAvailable: false, modelPresent: false, error: err.message };
}
this.renderLayaStatus(this.layaStatus);
const mode = this.app.player?.settings?.languageDetectionMode || 'algo';
if (mode !== 'algo' && this.layaStatus?.modelPresent && this.layaStatus?.runtimeAvailable &&
this.app.channelList?.languageDetectionNotice) {
void this.app.channelList.refreshLayaLanguages(mode);
}
if (this.layaStatus?.downloading) this.startLayaStatusPolling();
return this.layaStatus;
}

2
server/routes/live.js
View File

@ -71,7 +71,7 @@ router.get('/projection/status', requireAuth, (req, res) => {
router.post('/projection/rebuild', requireAuth, requireAdmin, (req, res) => {
try {
const started = liveProjection.startRebuild();
const started = liveProjection.startRebuild(null, { analyzeAll: true });
started.promise.catch(error => {
console.error('[LiveProjection] Manual rebuild failed:', error.message);
});

72
server/services/liveProjectionService.js
View File

@ -69,6 +69,22 @@ function stableHash(value) {
return crypto.createHash('sha256').update(value).digest('hex');
}
function languageInputHash(item) {
return stableHash(`${item.name || ''}\0${item.category_name || ''}\0${item.tvg_id || ''}`);
}
function hasPersistedLanguage(item) {
if (!/^[a-z]{2,3}$/i.test(item.language_code || '') || String(item.language_code).toLowerCase() === 'und') return false;
if (!item.language_detector || !item.language_input_hash) return false;
const persistedHash = String(item.language_input_hash);
return !/^[a-f0-9]{64}$/i.test(persistedHash) || persistedHash === languageInputHash(item);
}
function parseNullableData(value) {
if (value == null) return null;
return parseData(value);
}
function projectCatalog(items, options = {}) {
const candidateBuckets = new Map();
for (const item of items) {
@ -82,11 +98,22 @@ function projectCatalog(items, options = {}) {
qualityLabel: quality.label,
qualityRank: quality.rank
};
const language = detectLanguage({ ...projected, data: parseData(projected.data) });
projected.languageCode = language.code;
projected.languageConfidence = language.confidence;
projected.languageScores = language.scores;
projected.languageEvidence = language.evidence;
if (hasPersistedLanguage(item)) {
projected.languageCode = item.language_code;
projected.languageConfidence = item.language_confidence;
projected.languageScores = parseNullableData(item.language_scores);
projected.languageEvidence = parseNullableData(item.language_evidence);
projected.languageDetector = item.language_detector;
projected.languageInputHash = item.language_input_hash;
} else {
const language = detectLanguage({ ...projected, data: parseData(projected.data) });
projected.languageCode = language.code;
projected.languageConfidence = language.confidence;
projected.languageScores = language.scores;
projected.languageEvidence = language.evidence;
projected.languageDetector = 'algorithm';
projected.languageInputHash = languageInputHash(projected);
}
const itemIdentity = identity(fingerprint);
const candidateKey = itemIdentity.family && itemIdentity.number
? `${itemIdentity.family}:${itemIdentity.number}`
@ -124,7 +151,14 @@ function safeIdentityDescriptor(item) {
};
}
function identityCandidatePairs(items) {
function identityCandidatePairs(items, options = {}) {
const hasScope = Object.hasOwn(options, 'scopeItemIds');
const scopeItemIds = hasScope
? new Set(Array.from(options.scopeItemIds || [], String))
: null;
const isScoped = item => !scopeItemIds
|| scopeItemIds.has(String(item.id))
|| scopeItemIds.has(String(item.item_id));
const buckets = new Map();
for (const item of items) {
const fingerprint = normalizeChannelName(item.name);
@ -138,6 +172,7 @@ function identityCandidatePairs(items) {
for (const bucket of buckets.values()) {
for (let left = 0; left < bucket.length; left++) {
for (let right = left + 1; right < bucket.length; right++) {
if (!isScoped(bucket[left]) && !isScoped(bucket[right])) continue;
if (bucket[left].fingerprint === bucket[right].fingerprint) continue;
if (scoreCandidate(bucket[left], bucket[right], { mode: 'algo' }).blocker) continue;
pairs.push({
@ -154,7 +189,10 @@ function identityCandidatePairs(items) {
async function collectIdentityHints(items, mode, layaClient, options = {}) {
if (mode === 'algo' || !layaClient) return new Map();
const pairs = identityCandidatePairs(items);
const pairOptions = Object.hasOwn(options, 'scopeItemIds')
? { scopeItemIds: options.scopeItemIds }
: {};
const pairs = identityCandidatePairs(items, pairOptions);
const total = Math.ceil(pairs.length / 64);
let processed = 0;
const hints = new Map();
@ -227,7 +265,8 @@ function markProjectionBuilding(database = null) {
function loadProjectionItems(database) {
return database.prepare(`
SELECT p.id, p.source_id, p.item_id, p.name, p.category_id, p.data,
c.name AS category_name
p.language_code, p.language_confidence, p.language_scores, p.language_evidence,
p.language_detector, p.language_input_hash, c.name AS category_name
FROM playlist_items p
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id
WHERE p.type = 'live'
@ -252,9 +291,7 @@ function rebuild(database = null, options = {}) {
database.prepare('DELETE FROM logical_channels').run();
database.prepare(`
UPDATE playlist_items SET logical_channel_id = NULL, grouping_fingerprint = NULL,
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL,
language_code = NULL, language_confidence = NULL, language_scores = NULL,
language_evidence = NULL, language_detector = NULL, language_input_hash = NULL
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL
WHERE type = 'live'
`).run();
const insertLogical = database.prepare(`
@ -266,7 +303,7 @@ function rebuild(database = null, options = {}) {
UPDATE playlist_items SET logical_channel_id = ?, grouping_fingerprint = ?,
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?,
language_code = ?, language_confidence = ?, language_scores = ?,
language_evidence = ?, language_detector = 'algorithm', language_input_hash = ?
language_evidence = ?, language_detector = ?, language_input_hash = ?
WHERE id = ? AND type = 'live'
`);
const now = Date.now();
@ -276,8 +313,9 @@ function rebuild(database = null, options = {}) {
updateVariant.run(group.id, group.fingerprint, variant.groupingInputHash,
variant.qualityLabel, variant.qualityRank, variant.variantRank,
variant.languageCode, variant.languageConfidence,
JSON.stringify(variant.languageScores), JSON.stringify(variant.languageEvidence),
stableHash(`${variant.name || ''}\0${variant.category_name || ''}\0${variant.tvg_id || ''}`), variant.id);
variant.languageScores == null ? null : JSON.stringify(variant.languageScores),
variant.languageEvidence == null ? null : JSON.stringify(variant.languageEvidence),
variant.languageDetector, variant.languageInputHash, variant.id);
}
}
database.prepare(`
@ -307,7 +345,13 @@ async function rebuildWithSignals(database = null, options = {}) {
let identityHints = new Map();
if (mode !== 'algo') {
const layaClient = options.layaClient || new (require('./layaService').LayaService)();
const identityOptions = options.analyzeAll === true
? {}
: Object.hasOwn(options, 'scopeItemIds')
? { scopeItemIds: options.scopeItemIds }
: { scopeItemIds: items.filter(item => !hasPersistedLanguage(item)).map(item => item.id) };
identityHints = await collectOptionalIdentityHints(items, mode, layaClient, {
...identityOptions,
onProgress: ({ processed, total }) => updateProjectionProgress(database, 'laya_identity', {
layaProcessed: processed,
layaTotal: total

33
server/services/syncService.js
View File

@ -8,6 +8,18 @@ const liveProjection = require('./liveProjectionService');
// Sync tracking
const activeSyncs = new Set(); // sourceId
async function rebuildLiveProjection(newIds) {
const currentSettings = await settings.get();
const mode = currentSettings.languageDetectionMode;
if (newIds.size > 0 && (mode === 'laya' || mode === 'hybrid')) {
return liveProjection.startRebuild(undefined, {
mode,
scopeItemIds: [...newIds]
}).promise;
}
return liveProjection.queueRebuild();
}
class SyncService {
constructor() {
this.lastSyncTime = null; // Track when global sync last completed
@ -183,8 +195,8 @@ class SyncService {
// 2. Live Streams
console.log(`[Sync] Fetching Live Streams for ${source.name}`);
const liveStreams = await api.getLiveStreams();
await this.saveStreams(source.id, 'live', liveStreams);
await liveProjection.queueRebuild();
const { newIds } = await this.saveStreams(source.id, 'live', liveStreams);
await rebuildLiveProjection(newIds);
// 3. VOD Categories
console.log(`[Sync] Fetching VOD Categories for ${source.name}`);
@ -259,15 +271,17 @@ class SyncService {
* @param {string} type - Type of items (live, movie, series)
* @param {Array} items - Items to save
* @param {Object} options - Options { skipPurge: boolean }
* @returns {Set} Set of synced IDs (for external purge if skipPurge was true)
* @returns {{ syncedIds: Set<string>, newIds: Set<string> }} Synced and newly inserted IDs
*/
async saveStreams(sourceId, type, items, options = {}) {
if (!items || items.length === 0) return new Set();
if (!items || items.length === 0) return { syncedIds: new Set(), newIds: new Set() };
const db = getDb();
const { skipPurge = false } = options;
// Collect all IDs we're syncing
const syncedIds = new Set();
const newIds = new Set();
const existingItem = db.prepare('SELECT 1 FROM playlist_items WHERE id = ?');
const stmt = db.prepare(`
INSERT INTO playlist_items (
@ -316,6 +330,7 @@ class SyncService {
const id = `${sourceId}:${itemId}`;
syncedIds.add(id);
if (!existingItem.get(id)) newIds.add(id);
stmt.run(
id,
@ -349,7 +364,7 @@ class SyncService {
}
console.log(`[Sync] Saved ${items.length} ${type} items`);
return syncedIds;
return { syncedIds, newIds };
}
/**
@ -511,6 +526,7 @@ class SyncService {
const allGroups = new Set();
const allSyncedIds = new Set(); // Collect IDs across all batches
const allNewIds = new Set();
let totalChannels = 0;
let batchCount = 0;
@ -530,8 +546,9 @@ class SyncService {
// Save this batch immediately (skip purge - we'll do it at the end)
if (playlistItems.length > 0) {
const batchIds = await this.saveStreams(source.id, 'live', playlistItems, { skipPurge: true });
batchIds.forEach(id => allSyncedIds.add(id));
const { syncedIds, newIds } = await this.saveStreams(source.id, 'live', playlistItems, { skipPurge: true });
syncedIds.forEach(id => allSyncedIds.add(id));
newIds.forEach(id => allNewIds.add(id));
totalChannels += playlistItems.length;
}
@ -561,7 +578,7 @@ class SyncService {
}));
await this.saveCategories(source.id, 'live', categories);
await liveProjection.queueRebuild();
await rebuildLiveProjection(allNewIds);
console.log(`[Sync] M3U sync complete for ${source.name}`);
}

54
test/liveAnalysisUserFlowDom.test.js
View File

@ -141,12 +141,12 @@ test('Settings renders Laya batch progress and removes misleading values when it
assert.equal(bar.style.width, '');
});
test('analysis remains visibly busy while Laya language batches run after projection completes', async () => {
test('analysis completion reloads the persisted projection', async () => {
const { window } = createDom();
let rebuildClicked = false;
let languageRefreshStarted = false;
let releaseLanguageRefresh;
const languageRefresh = new Promise(resolve => { releaseLanguageRefresh = resolve; });
let projectionReloadStarted = false;
let releaseProjectionReload;
const projectionReload = new Promise(resolve => { releaseProjectionReload = resolve; });
window.API = {
live: {
getProjectionStatus: async () => rebuildClicked
@ -176,11 +176,11 @@ test('analysis remains visibly busy while Laya language batches run after projec
saveSettings() {}
},
channelList: {
loadChannels: async () => {},
refreshLayaLanguages: async () => {
languageRefreshStarted = true;
await languageRefresh;
}
loadChannels: async () => {
projectionReloadStarted = true;
await projectionReload;
},
refreshLayaLanguages: async () => assert.fail('Browser-side Laya refresh must not run')
}
};
window.app = app;
@ -193,28 +193,19 @@ test('analysis remains visibly busy while Laya language batches run after projec
window.document.getElementById('live-analysis-button').click();
try {
await waitFor(() => languageRefreshStarted, 'Expected Laya language refresh to start after projection completion');
assert.notEqual(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete');
assert.equal(window.document.getElementById('live-analysis-progress').hidden, false);
await waitFor(() => projectionReloadStarted, 'Expected persisted projection reload after analysis completion');
assert.equal(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete');
} finally {
releaseLanguageRefresh();
releaseProjectionReload();
window.clearInterval(settingsPage.liveAnalysisPollTimer);
}
});
test('user analysis click keeps visible channels after Local AI language failure', async () => {
test('user analysis click reloads persisted languages without a browser-side Laya batch', async () => {
const { window } = createDom();
let rebuildClicked = 0;
let layaBatchCalls = 0;
let channelLoads = 0;
const consoleWarnings = [];
window.console = {
...console,
warn: (...args) => {
consoleWarnings.push(args.join(' '));
console.warn(...args);
}
};
window.API = {
live: {
getChannels: async () => {
@ -269,26 +260,22 @@ test('user analysis click keeps visible channels after Local AI language failure
window.clearInterval(channelList._epgRefreshTimer);
window.clearInterval(settingsPage.liveAnalysisPollTimer);
const notice = window.document.querySelector('.laya-language-notice');
const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent);
const languageBadges = [...window.document.querySelectorAll('.variant-language')].map(item => item.textContent);
const groupNames = [...window.document.querySelectorAll('.group-name')].map(item => item.textContent);
assert.equal(channelLoads, 2);
assert.equal(layaBatchCalls, 1);
assert.ok(notice);
assert.match(notice.textContent, /Local AI language detection failed/);
assert.equal(layaBatchCalls, 0);
assert.deepEqual(afterNames, ['Sports One', 'Sports One']);
assert.deepEqual([...new Set(languageBadges)].sort(), ['EN 100%', 'FR 100%']);
assert.deepEqual(groupNames, ['Favorites', 'Sports France']);
assert.equal(window.document.querySelector('.empty-state'), null);
assert.equal(window.document.querySelector('#channel-list > .loading'), null);
assert.equal(channelList.languageDetectionFallback, true);
assert.equal(channelList.languageDetectionFallback, false);
assert.equal(channelList.logicalChannels[0].variants.length, 2);
assert.ok(consoleWarnings.some(line => line.includes('Local AI language detection unavailable')));
});
test('successful Standard fallback after Local AI failure keeps channels and reports the AI failure', async () => {
test('hybrid analysis completion relies on persisted projection without Standard browser fallback', async () => {
const { window } = createDom();
let rebuildClicked = 0;
let layaBatchCalls = 0;
@ -345,16 +332,13 @@ test('successful Standard fallback after Local AI failure keeps channels and rep
window.clearInterval(settingsPage.liveAnalysisPollTimer);
const afterNames = [...window.document.querySelectorAll('.channel-name')].map(item => item.textContent);
const notice = window.document.querySelector('.laya-language-notice');
assert.equal(layaBatchCalls, 1);
assert.equal(layaBatchCalls, 0);
assert.deepEqual(afterNames, ['Sports One', 'Sports One']);
assert.equal(window.document.querySelector('.empty-state'), null);
assert.equal(window.document.querySelector('#channel-list > .loading'), null);
assert.ok(notice);
assert.match(notice.textContent, /Local AI language detection failed/);
assert.match(notice.textContent, /Standard matching/);
assert.equal(channelList.languageDetectionFallback, true);
assert.equal(window.document.querySelector('.laya-language-notice'), null);
assert.equal(channelList.languageDetectionFallback, false);
});
test('successful Laya language batch updates visible variant languages without Standard fallback', async () => {

4
test/liveProjectionRoutes.test.js
View File

@ -15,6 +15,10 @@ test('projection rebuild route requires authentication and admin authorization',
assert.match(routeSource, /router\.post\('\/projection\/rebuild', requireAuth, requireAdmin,/);
});
test('projection rebuild route requests a complete analysis', () => {
assert.match(routeSource, /liveProjection\.startRebuild\(null, \{ analyzeAll: true \}\)/);
});
test('projection authorization distinguishes viewer and admin roles', () => {
assert.equal(isAdmin({ role: 'viewer' }), false);
assert.equal(isAdmin({ role: 'admin' }), true);

31
test/t25Frontend.test.js
View File

@ -25,6 +25,7 @@ function createChannelList() {
window.eval(script('js/services/channelLanguageDetection.js'));
window.eval(script('js/services/channelLanguageFusion.js'));
window.eval(script('js/services/channelVariantGrouping.js'));
window.eval(script('js/services/layaStatusPresentation.js'));
window.eval(script('js/services/liveChannelProjection.js'));
window.eval(`${script('js/components/ChannelList.js')}\nwindow.ChannelList = ChannelList;`);
window.app = {
@ -95,6 +96,36 @@ test('language selection filters the complete loaded catalog immediately without
dom.window.close();
});
test('loading the projected channel list does not call Laya language detection during display', async () => {
const { dom, window, channelList } = createChannelList();
window.app.player.settings.languageDetectionMode = 'laya';
let layaStatusCalls = 0;
let layaBatchCalls = 0;
window.API = {
live: { getChannels: async () => projectedResponse() },
laya: {
getStatus: async () => {
layaStatusCalls += 1;
return { modelPresent: true, runtimeAvailable: true };
},
detectLanguages: async () => {
layaBatchCalls += 1;
return { results: [] };
}
}
};
try {
await channelList.loadAllChannels();
await wait(20);
assert.equal(layaStatusCalls, 0);
assert.equal(layaBatchCalls, 0);
} finally {
dom.window.close();
}
});
test('Escape clears channel search while preserving language-panel Escape behavior', async () => {
const { dom, window, channelList } = createChannelList();
window.API = { live: { getChannels: async () => projectedResponse() } };

171
test/t36LanguagePersistenceRegression.test.js Normal file
View File

@ -0,0 +1,171 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const Database = require('better-sqlite3');
const {
ensureColumns,
LIVE_PROJECTION_COLUMNS,
LIVE_PROJECTION_SCHEMA_SQL
} = require('../server/db/liveProjectionSchema');
const liveProjection = require('../server/services/liveProjectionService');
function createDb() {
const db = new Database(':memory:');
db.exec(`
CREATE TABLE categories (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
category_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
parent_id TEXT,
is_hidden INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE playlist_items (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
category_id TEXT,
parent_id TEXT,
stream_icon TEXT,
stream_url TEXT,
container_extension TEXT,
rating REAL,
year TEXT,
added_at TEXT,
is_hidden INTEGER DEFAULT 0,
is_favorite INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE favorites (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
item_type TEXT NOT NULL
);
`);
ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
db.exec(LIVE_PROJECTION_SCHEMA_SQL);
return db;
}
function seedChannel(db, id, name, extra = {}) {
db.prepare(`
INSERT INTO playlist_items
(id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data,
language_code, language_confidence, language_detector, language_input_hash)
VALUES (?, 1, ?, 'live', ?, 'general', '', ?, '{}', ?, ?, ?, ?)
`).run(
`1:${id}`,
id,
name,
`https://streams.test/${id}.m3u8`,
extra.languageCode || null,
extra.languageConfidence || null,
extra.languageDetector || null,
extra.languageInputHash || null
);
}
test('background projection rebuild keeps a persisted stream language instead of redetecting it', () => {
const db = createDb();
seedChannel(db, 'persisted', 'World News', {
languageCode: 'fr',
languageConfidence: 0.99,
languageDetector: 'laya',
languageInputHash: 'stable-language-input'
});
try {
liveProjection.rebuild(db, { mode: 'algo' });
const row = db.prepare(`
SELECT language_code, language_detector, language_confidence
FROM playlist_items
WHERE id = '1:persisted'
`).get();
assert.equal(row.language_code, 'fr');
assert.equal(row.language_detector, 'laya');
assert.equal(Number(row.language_confidence), 0.99);
} finally {
db.close();
}
});
test('Laya analysis for one newly detected stream does not send old catalog pairs', async () => {
const db = createDb();
for (let index = 0; index < 8; index++) {
seedChannel(db, `old-${index}`, `beIN Sports 1 legacy ${index}`, {
languageCode: 'fr',
languageConfidence: 0.95,
languageDetector: 'laya',
languageInputHash: `old-${index}-hash`
});
}
seedChannel(db, 'new-stream', 'beIN Sports 1 new feed');
const oldCatalogPairs = [];
const layaClient = {
identityHintBatch: async pairs => {
for (const pair of pairs) {
if (!pair.left.name.includes('new feed') && !pair.right.name.includes('new feed')) {
oldCatalogPairs.push(pair);
}
}
return pairs.map(pair => ({
id: pair.id,
sameIdentityProbability: 0.9,
confidence: 0.9,
family: 'bein sports',
number: '1'
}));
}
};
try {
await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient });
assert.equal(oldCatalogPairs.length, 0, `unexpected old-catalog Laya pairs=${oldCatalogPairs.length}`);
} finally {
db.close();
}
});
test('analyzeAll sends persisted catalog pairs to Laya', async () => {
const db = createDb();
seedChannel(db, 'persisted-a', 'beIN Sports 1 east', {
languageCode: 'fr',
languageConfidence: 0.95,
languageDetector: 'laya',
languageInputHash: 'persisted-a-hash'
});
seedChannel(db, 'persisted-b', 'beIN Sports 1 west', {
languageCode: 'fr',
languageConfidence: 0.95,
languageDetector: 'laya',
languageInputHash: 'persisted-b-hash'
});
let layaPairs = 0;
const layaClient = {
identityHintBatch: async pairs => {
layaPairs += pairs.length;
return pairs.map(pair => ({
id: pair.id,
sameIdentityProbability: 0.9,
confidence: 0.9,
family: 'bein sports',
number: '1'
}));
}
};
try {
await liveProjection.rebuildWithSignals(db, { mode: 'laya', layaClient, analyzeAll: true });
assert.ok(layaPairs > 0);
} finally {
db.close();
}
});

201
test/t36SyncIncrementalRegression.test.js Normal file
View File

@ -0,0 +1,201 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const Database = require('better-sqlite3');
function createDb() {
const db = new Database(':memory:');
db.exec(`
CREATE TABLE categories (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
category_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
parent_id TEXT,
is_hidden INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE playlist_items (
id TEXT PRIMARY KEY,
source_id INTEGER NOT NULL,
item_id TEXT NOT NULL,
type TEXT NOT NULL,
name TEXT NOT NULL,
category_id TEXT,
parent_id TEXT,
stream_icon TEXT,
stream_url TEXT,
container_extension TEXT,
rating REAL,
year TEXT,
added_at TEXT,
is_hidden INTEGER DEFAULT 0,
is_favorite INTEGER DEFAULT 0,
data JSON
);
CREATE TABLE sync_status (
source_id INTEGER NOT NULL,
type TEXT NOT NULL,
last_sync INTEGER NOT NULL,
status TEXT,
error TEXT,
PRIMARY KEY (source_id, type)
);
`);
return db;
}
function seedLive(db, sourceId, streamId, name = `Channel ${streamId}`) {
db.prepare(`
INSERT INTO playlist_items
(id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data)
VALUES (?, ?, ?, 'live', ?, 'sports', '', ?, '{}')
`).run(`${sourceId}:${streamId}`, sourceId, String(streamId), name, `https://streams.test/${streamId}.m3u8`);
}
function loadSyncServiceWithDb(db) {
const sqlite = require('../server/db/sqlite');
const originalGetDb = sqlite.getDb;
sqlite.getDb = () => db;
const syncServicePath = require.resolve('../server/services/syncService');
delete require.cache[syncServicePath];
const syncService = require('../server/services/syncService');
return {
syncService,
restore() {
sqlite.getDb = originalGetDb;
delete require.cache[syncServicePath];
}
};
}
function stream(streamId, name = `Channel ${streamId}`) {
return {
stream_id: streamId,
name,
category_id: 'sports',
stream_icon: '',
added: 'now'
};
}
function patchLiveProjection() {
const liveProjection = require('../server/services/liveProjectionService');
const originalQueueRebuild = liveProjection.queueRebuild;
const originalStartRebuild = liveProjection.startRebuild;
const requests = [];
liveProjection.queueRebuild = async (...args) => {
requests.push({ method: 'queueRebuild', args });
return { status: 'success' };
};
liveProjection.startRebuild = (...args) => {
requests.push({ method: 'startRebuild', args });
return { promise: Promise.resolve({ status: 'success' }) };
};
return {
requests,
restore() {
liveProjection.queueRebuild = originalQueueRebuild;
liveProjection.startRebuild = originalStartRebuild;
}
};
}
function patchLanguageMode(mode) {
const appDb = require('../server/db');
const originalGet = appDb.settings.get;
appDb.settings.get = async () => ({ ...appDb.getDefaultSettings(), languageDetectionMode: mode });
return () => { appDb.settings.get = originalGet; };
}
test('saveStreams returns only newly inserted live stream ids, not updated ids', async () => {
const db = createDb();
seedLive(db, 1, 'old', 'Old Sports');
const { syncService, restore } = loadSyncServiceWithDb(db);
try {
const result = await syncService.saveStreams(1, 'live', [
stream('old', 'Old Sports Updated'),
stream('new', 'New Sports')
], { skipPurge: true });
const newIds = result?.newIds || new Set();
assert.deepEqual([...newIds].sort(), ['1:new']);
} finally {
restore();
db.close();
}
});
test('syncXtream with new live ids requests one targeted analysis for only those ids', async () => {
const db = createDb();
seedLive(db, 1, 'old', 'Old Sports');
const { syncService, restore } = loadSyncServiceWithDb(db);
const liveProjectionPatch = patchLiveProjection();
const restoreLanguageMode = patchLanguageMode('laya');
const xtreamApi = require('../server/services/xtreamApi');
const originalCreateFromSource = xtreamApi.createFromSource;
xtreamApi.createFromSource = () => ({
getLiveCategories: async () => [{ category_id: 'sports', category_name: 'Sports' }],
getLiveStreams: async () => [stream('old', 'Old Sports Updated'), stream('new', 'New Sports')],
getVodCategories: async () => [],
getVodStreams: async () => [],
getSeriesCategories: async () => [],
getSeries: async () => [],
getXmltvUrl: () => { throw new Error('no epg'); }
});
try {
await syncService.syncXtream({ id: 1, name: 'Xtream Fixture' });
assert.deepEqual(liveProjectionPatch.requests, [{
method: 'startRebuild',
args: [undefined, { mode: 'laya', scopeItemIds: ['1:new'] }]
}]);
} finally {
xtreamApi.createFromSource = originalCreateFromSource;
restoreLanguageMode();
liveProjectionPatch.restore();
restore();
db.close();
}
});
test('syncM3u aggregates new ids from all batches and requests one final targeted analysis', async () => {
const db = createDb();
seedLive(db, 1, 'old-a', 'Old A');
seedLive(db, 1, 'old-b', 'Old B');
const { syncService, restore } = loadSyncServiceWithDb(db);
const liveProjectionPatch = patchLiveProjection();
const restoreLanguageMode = patchLanguageMode('laya');
const m3uParser = require('../server/services/m3uParser');
const originalFetchAndParseStreaming = m3uParser.fetchAndParseStreaming;
m3uParser.fetchAndParseStreaming = async function* () {
yield {
channels: [
{ id: 'old-a', name: 'Old A Updated', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/old-a.m3u8' },
{ id: 'new-a', name: 'New A', groupTitle: 'Sports', tvgLogo: '', url: 'https://streams.test/new-a.m3u8' }
],
groups: ['Sports']
};
yield {
channels: [
{ id: 'old-b', name: 'Old B Updated', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/old-b.m3u8' },
{ id: 'new-b', name: 'New B', groupTitle: 'News', tvgLogo: '', url: 'https://streams.test/new-b.m3u8' }
],
groups: ['News']
};
};
try {
await syncService.syncM3u({ id: 1, name: 'M3U Fixture', url: 'https://playlist.test/live.m3u' });
assert.deepEqual(liveProjectionPatch.requests, [{
method: 'startRebuild',
args: [undefined, { mode: 'laya', scopeItemIds: ['1:new-a', '1:new-b'] }]
}]);
} finally {
m3uParser.fetchAndParseStreaming = originalFetchAndParseStreaming;
restoreLanguageMode();
liveProjectionPatch.restore();
restore();
db.close();
}
});