diff --git a/public/js/components/ChannelList.js b/public/js/components/ChannelList.js index 86eb9e9..a2962b7 100644 --- a/public/js/components/ChannelList.js +++ b/public/js/components/ChannelList.js @@ -344,7 +344,7 @@ class ChannelList { } } - async refreshLayaLanguages(mode = window.app?.player?.settings?.languageDetectionMode || 'algo') { + async refreshLayaLanguages(mode = window.app?.player?.settings?.languageDetectionMode || 'algo', options = {}) { const requestId = ++this._languageDetectionRequest; if (mode === 'algo') { const hadWarning = Boolean(this.languageDetectionNotice || this.languageDetectionFallback); @@ -364,6 +364,7 @@ class ChannelList { ); }); const results = new Map(); + const totalBatches = Math.ceil(descriptors.length / 64); try { const status = await API.laya.getStatus(); const statusWarning = LayaStatusPresentation.channelListWarning(mode, status); @@ -379,9 +380,15 @@ class ChannelList { this.languageDetectionNotice = ''; this.languageDetectionFallback = false; this.render(); + options.onProgress?.({ processed: 0, total: totalBatches }); for (let index = 0; index < descriptors.length; index += 64) { const response = await API.laya.detectLanguages(descriptors.slice(index, index + 64)); (response.results || []).forEach(result => results.set(String(result.id), result)); + if (requestId !== this._languageDetectionRequest) return; + options.onProgress?.({ + processed: Math.floor(index / 64) + 1, + total: totalBatches + }); } if (requestId !== this._languageDetectionRequest) return; if (descriptors.length > 0 && results.size === 0) { @@ -904,7 +911,7 @@ class ChannelList { /** * Load channels from selected source */ - async loadChannels() { + async loadChannels({ refreshLanguages = true } = {}) { if (this.isLoading) return; this.isLoading = true; this.currentRenderId = null; // Reset render tracking @@ -921,7 +928,7 @@ class ChannelList { if (!sourceValue) { // Load from all sources - await this.loadAllChannels(); + await this.loadAllChannels({ refreshLanguages }); this.isLoading = false; return; } @@ -933,7 +940,7 @@ class ChannelList { if (await this.loadProjectedChannels(parseInt(id, 10))) { this.render(); - void this.refreshLayaLanguages(); + if (refreshLanguages) void this.refreshLayaLanguages(); return; } @@ -950,7 +957,7 @@ class ChannelList { ]); this.render(); - void this.refreshLayaLanguages(); + if (refreshLanguages) void this.refreshLayaLanguages(); } catch (err) { console.error('Error loading channels:', err); if (previousState.channels.length > 0) { @@ -970,7 +977,7 @@ class ChannelList { /** * Load channels from all enabled sources */ - async loadAllChannels() { + async loadAllChannels({ refreshLanguages = true } = {}) { const previousState = { channels: this.channels, groups: this.groups, @@ -985,7 +992,7 @@ class ChannelList { if (await this.loadProjectedChannels()) { this.render(); - void this.refreshLayaLanguages(); + if (refreshLanguages) void this.refreshLayaLanguages(); return; } @@ -1006,7 +1013,7 @@ class ChannelList { this.loadFavorites() ]); this.render(); - void this.refreshLayaLanguages(); + if (refreshLanguages) void this.refreshLayaLanguages(); } catch (err) { console.error('Error loading all channels:', err); if (previousState.channels.length > 0) { diff --git a/public/js/pages/Settings.js b/public/js/pages/Settings.js index a3d664c..9dfba4d 100644 --- a/public/js/pages/Settings.js +++ b/public/js/pages/Settings.js @@ -197,11 +197,32 @@ class SettingsPage { clearInterval(this.liveAnalysisPollTimer); this.liveAnalysisPollTimer = null; if (presentation.complete) { - await this.app.channelList?.loadChannels(); const mode = this.app.player?.settings?.languageDetectionMode || 'algo'; + await this.app.channelList?.loadChannels({ + refreshLanguages: mode !== 'laya' && mode !== 'hybrid' + }); if (mode === 'laya' || mode === 'hybrid') { - await this.app.channelList?.refreshLayaLanguages(mode); + this.liveAnalysisStatus = { + status: 'building', + phase: 'laya_language', + languageProcessed: null, + languageTotal: null + }; + this.renderLiveAnalysisStatus(this.liveAnalysisStatus); + await this.app.channelList?.refreshLayaLanguages(mode, { + onProgress: ({ processed, total }) => { + this.liveAnalysisStatus = { + status: 'building', + phase: 'laya_language', + languageProcessed: processed, + languageTotal: total + }; + this.renderLiveAnalysisStatus(this.liveAnalysisStatus); + } + }); } + this.liveAnalysisStatus = status; + this.renderLiveAnalysisStatus(status); } } } finally { diff --git a/public/js/services/channelLanguageFusion.js b/public/js/services/channelLanguageFusion.js index 0ebfeb8..f09fd8d 100644 --- a/public/js/services/channelLanguageFusion.js +++ b/public/js/services/channelLanguageFusion.js @@ -10,9 +10,19 @@ return Math.max(0, Math.min(100, Math.round(number <= 1 ? number * 100 : number))); } + function normalizeLanguageCode(value) { + const normalized = String(value || '').trim().toLowerCase(); + const aliases = { + fr: 'fr', french: 'fr', français: 'fr', francais: 'fr', + en: 'en', english: 'en', anglais: 'en', + es: 'es', spanish: 'es', español: 'es', espanol: 'es' + }; + return aliases[normalized] || 'unknown'; + } + function layaDetection(result) { - const rawCode = result?.language?.code || result?.language || 'unknown'; - const code = ['fr', 'en', 'es'].includes(rawCode) ? rawCode : 'unknown'; + const rawCode = result?.language?.code || result?.language?.label || result?.language || 'unknown'; + const code = normalizeLanguageCode(rawCode); const confidence = normalizeConfidence(result?.language?.confidence ?? result?.confidence); return { language: { @@ -79,5 +89,5 @@ }; } - return { applyLanguageMode, buildLayaDescriptor, layaDetection, mergeHybrid, normalizeConfidence }; + return { applyLanguageMode, buildLayaDescriptor, layaDetection, mergeHybrid, normalizeConfidence, normalizeLanguageCode }; }); diff --git a/public/js/services/liveAnalysisPresentation.js b/public/js/services/liveAnalysisPresentation.js index cae84f8..fd6e2d7 100644 --- a/public/js/services/liveAnalysisPresentation.js +++ b/public/js/services/liveAnalysisPresentation.js @@ -14,6 +14,13 @@ prefix: 'AI batch' }; } + if (phase === 'laya_language') { + return { + processed: status.languageProcessed, + total: status.languageTotal, + prefix: 'Language batch' + }; + } return { processed: status.processed, total: status.total, prefix: '', suffix: ' channels' }; } @@ -37,6 +44,8 @@ const statusLabel = busy ? phase === 'laya_identity' ? 'Analyzing with local AI…' + : phase === 'laya_language' + ? 'Detecting stream languages…' : phase === 'projecting' ? 'Building channel catalog…' : phase === 'writing' diff --git a/server/services/layaService.js b/server/services/layaService.js index b1f82a3..c12f024 100644 --- a/server/services/layaService.js +++ b/server/services/layaService.js @@ -9,6 +9,13 @@ const MAX_TEXT_LENGTH = 4096; const ALLOWED_MODELS = new Set(['english', 'multilingual', 'typed-decisions']); const DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000; const BATCH_TIMEOUT_MS = 2 * 60 * 1000; +const LANGUAGE_CACHE_VERSION = 'v2'; +const LANGUAGE_LABELS = new Map([ + ['fr', 'fr'], ['fr-fr', 'fr'], ['french', 'fr'], ['francais', 'fr'], ['francaise', 'fr'], + ['en', 'en'], ['en-gb', 'en'], ['en-us', 'en'], ['english', 'en'], ['anglais', 'en'], + ['es', 'es'], ['es-es', 'es'], ['spanish', 'es'], ['espanol', 'es'], ['espagnol', 'es'], ['castellano', 'es'], + ['other', 'other'], ['another language', 'other'], ['unknown', 'unknown'], ['undetermined', 'unknown'] +]); class LayaRequestError extends Error { constructor(message, statusCode = 503) { @@ -120,6 +127,34 @@ function sanitizeIdentityDescriptor(value) { }; } +function normalizeLanguageLabel(value) { + if (typeof value !== 'string' || !value.trim()) return 'unknown'; + const normalized = value + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .trim() + .toLowerCase() + .replace(/_/g, '-'); + return LANGUAGE_LABELS.get(normalized) || 'unknown'; +} + +function parseLanguageAnswer(answer) { + const choice = typeof answer === 'string' + ? answer + : answer?.choice ?? answer?.value ?? answer?.label; + if (typeof choice !== 'string' || !choice.trim()) { + throw new LayaRequestError('Laya runtime returned an invalid language answer', 502); + } + const confidenceValue = typeof answer === 'object' + ? answer?.answer_confidence ?? answer?.confidence + : 0; + const confidence = Number(confidenceValue); + return { + language: normalizeLanguageLabel(choice), + confidence: Number.isFinite(confidence) ? confidence : 0 + }; +} + class LayaService { constructor(options = {}) { this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000'; @@ -291,8 +326,8 @@ class LayaService { } } - cacheKey(text, mode, model) { - return crypto.createHash('sha256').update(`${mode}\0${model}\0${text}`).digest('hex'); + cacheKey(text, mode, model, version = '') { + return crypto.createHash('sha256').update(`${version}\0${mode}\0${model}\0${text}`).digest('hex'); } async detectBatch(payload) { @@ -303,7 +338,7 @@ class LayaService { const misses = []; payload.items.forEach((item, index) => { - const key = this.cacheKey(item.text, mode, model); + const key = this.cacheKey(item.text, mode, model, LANGUAGE_CACHE_VERSION); const cached = this.cache ? this.cache.get(key) : null; if (cached) results[index] = { ...cached, cached: true }; else misses.push({ item, index, key }); @@ -328,12 +363,11 @@ class LayaService { if (!response || !Array.isArray(response.results) || response.results.length !== misses.length) { throw new LayaRequestError('Laya runtime returned an invalid batch response', 502); } + const parsed = response.results.map(result => parseLanguageAnswer(result?.answers?.language)); misses.forEach(({ item, index, key }, responseIndex) => { - const answer = response.results[responseIndex]?.answers?.language; const result = { id: item.id ?? index, - language: answer?.choice || 'unknown', - confidence: answer?.answer_confidence ?? answer?.confidence ?? 0, + ...parsed[responseIndex], cached: false }; results[index] = result; @@ -389,6 +423,8 @@ module.exports = { clearDirectoryContents, directorySize, initialDownloadState, + normalizeLanguageLabel, + parseLanguageAnswer, sanitizeIdentityDescriptor, validateBatchPayload }; diff --git a/test/channelLanguageFusion.test.js b/test/channelLanguageFusion.test.js index 33417c0..0562d08 100644 --- a/test/channelLanguageFusion.test.js +++ b/test/channelLanguageFusion.test.js @@ -2,6 +2,12 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const fusion = require('../public/js/services/channelLanguageFusion.js'); +test('normalizes common Laya language labels to supported codes', () => { + assert.equal(fusion.normalizeLanguageCode('French'), 'fr'); + assert.equal(fusion.normalizeLanguageCode('English'), 'en'); + assert.equal(fusion.normalizeLanguageCode('ESPAÑOL'), 'es'); +}); + const algo = { language: { code: 'fr', label: 'French', confidence: 70, status: 'medium' }, languageScores: { fr: 70, en: 30 }, diff --git a/test/layaService.test.js b/test/layaService.test.js index e1c2f59..7d4ea3b 100644 --- a/test/layaService.test.js +++ b/test/layaService.test.js @@ -152,8 +152,8 @@ test('language batch sends the Laya contract and parses language confidence with ok: true, json: async () => ({ results: [ - { answers: { language: { choice: 'fr', answer_confidence: 0.93 } } }, - { answers: { language: { choice: 'en', confidence: 0.81 } } } + { answers: { language: { choice: 'French', answer_confidence: 0.93 } } }, + { answers: { language: { choice: 'English', confidence: 0.81 } } } ] }) }; @@ -181,6 +181,45 @@ test('language batch sends the Laya contract and parses language confidence with }); }); +test('language batch preserves explicit other and maps an unfamiliar label to unknown', async () => { + const service = new LayaService({ + cache: null, + fetch: async () => ({ + ok: true, + json: async () => ({ + results: [ + { answers: { language: { choice: 'Other', confidence: 0.72 } } }, + { answers: { language: { choice: 'Klingon', confidence: 0.99 } } } + ] + }) + }) + }); + + const response = await service.detectBatch({ + items: [{ id: 'other', text: 'Autre' }, { id: 'strange', text: 'Qapla' }] + }); + + assert.deepEqual(response.results, [ + { id: 'other', language: 'other', confidence: 0.72, cached: false }, + { id: 'strange', language: 'unknown', confidence: 0.99, cached: false } + ]); +}); + +test('language batch rejects a successful HTTP response without a usable language answer', async () => { + const service = new LayaService({ + cache: null, + fetch: async () => ({ + ok: true, + json: async () => ({ results: [{ answers: { language: { confidence: 0.9 } } }] }) + }) + }); + + await assert.rejects( + service.detectBatch({ items: [{ id: 'missing', text: 'Missing language' }] }), + error => error instanceof LayaRequestError && error.statusCode === 502 + ); +}); + test('identity hints send only sanitized channel metadata to Laya', async () => { let requestBody; const service = new LayaService({ diff --git a/test/liveAnalysisPresentation.test.js b/test/liveAnalysisPresentation.test.js index 0cc0e2f..513daaf 100644 --- a/test/liveAnalysisPresentation.test.js +++ b/test/liveAnalysisPresentation.test.js @@ -26,6 +26,19 @@ test('presents Laya identity progress using AI batch counters', () => { assert.equal(view.progressText, 'AI batch 3 of 8 · 38%'); }); +test('presents local Laya language batch progress after projection', () => { + const view = present({ + status: 'building', + phase: 'laya_language', + languageProcessed: 5, + languageTotal: 10 + }); + assert.equal(view.statusLabel, 'Detecting stream languages…'); + assert.equal(view.progressIndeterminate, false); + assert.equal(view.progressPercent, 50); + assert.equal(view.progressText, 'Language batch 5 of 10 · 50%'); +}); + test('presents indeterminate progress when totals are unavailable', () => { const view = present({ status: 'queued', processed: null, total: null }); assert.equal(view.busy, true); diff --git a/test/liveAnalysisUserFlowDom.test.js b/test/liveAnalysisUserFlowDom.test.js index da0d728..1afcc5e 100644 --- a/test/liveAnalysisUserFlowDom.test.js +++ b/test/liveAnalysisUserFlowDom.test.js @@ -92,6 +92,18 @@ function projectedChannels() { }; } +function waitFor(condition, message) { + return new Promise((resolve, reject) => { + const started = Date.now(); + const poll = () => { + if (condition()) return resolve(); + if (Date.now() - started > 1500) return reject(new Error(message)); + setTimeout(poll, 5); + }; + poll(); + }); +} + test('Settings renders Laya batch progress and removes misleading values when its total is unknown', () => { const { window } = createDom(); const settingsPage = Object.create(window.SettingsPage.prototype); @@ -129,6 +141,67 @@ test('Settings renders Laya batch progress and removes misleading values when it assert.equal(bar.style.width, ''); }); +test('analysis remains visibly busy while Laya language batches run after projection completes', async () => { + const { window } = createDom(); + let rebuildClicked = false; + let languageRefreshStarted = false; + let releaseLanguageRefresh; + const languageRefresh = new Promise(resolve => { releaseLanguageRefresh = resolve; }); + window.API = { + live: { + getProjectionStatus: async () => rebuildClicked + ? { status: 'success', phase: 'complete', processed: 128, total: 128, projectedCount: 64 } + : { status: 'idle', processed: 0, total: 128 }, + rebuildProjection: async () => { + rebuildClicked = true; + return { + projection: { + status: 'building', + phase: 'laya_identity', + layaProcessed: 4, + layaTotal: 4, + processed: 0, + total: 128 + } + }; + } + }, + laya: { + getStatus: async () => ({ modelPresent: true, runtimeAvailable: true }) + } + }; + const app = { + player: { + settings: { languageDetectionMode: 'laya' }, + saveSettings() {} + }, + channelList: { + loadChannels: async () => {}, + refreshLayaLanguages: async () => { + languageRefreshStarted = true; + await languageRefresh; + } + } + }; + window.app = app; + const settingsPage = Object.create(window.SettingsPage.prototype); + settingsPage.app = app; + settingsPage.layaStatus = { modelPresent: true, runtimeAvailable: true }; + + settingsPage.initLiveAnalysis(); + await Promise.resolve(); + window.document.getElementById('live-analysis-button').click(); + + try { + await waitFor(() => languageRefreshStarted, 'Expected Laya language refresh to start after projection completion'); + assert.notEqual(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete'); + assert.equal(window.document.getElementById('live-analysis-progress').hidden, false); + } finally { + releaseLanguageRefresh(); + window.clearInterval(settingsPage.liveAnalysisPollTimer); + } +}); + test('user analysis click keeps visible channels after Local AI language failure', async () => { const { window } = createDom(); let rebuildClicked = 0; @@ -348,6 +421,53 @@ test('successful Laya language batch updates visible variant languages without S assert.equal(window.document.querySelector('.laya-language-notice'), null); }); +test('usable Laya language labels do not leave visible variants as unknown', async () => { + const { window } = createDom(); + window.API = { + live: { + getChannels: async () => projectedChannels() + }, + laya: { + getStatus: async () => ({ modelPresent: true, runtimeAvailable: true }), + detectLanguages: async items => ({ + runtimeAvailable: true, + model: 'multilingual', + results: items.map(item => ({ + id: item.id, + language: item.id.endsWith('fr-sd') ? 'French' : 'English', + confidence: 0.91 + })) + }) + } + }; + + const app = { + player: { + settings: { liveChannelListMode: 'grouped', languageDetectionMode: 'laya' }, + saveSettings() {} + }, + epgGuide: null, + navigateTo() {}, + pages: { settings: { switchTab() {} } } + }; + window.app = app; + const channelList = new window.ChannelList(); + channelList.sources = [{ id: 1, type: 'm3u', name: 'Fixture Provider' }]; + app.channelList = channelList; + + await channelList.loadChannels(); + await channelList.refreshLayaLanguages('laya'); + channelList.expandAll(); + + const badges = [...window.document.querySelectorAll('.variant-language')].map(item => item.textContent); + + window.clearInterval(channelList._epgRefreshTimer); + + assert.deepEqual([...new Set(badges)].sort(), ['EN 91%', 'FR 91%']); + assert.equal(channelList.languageDetectionFallback, false); + assert.equal(channelList.languageDetectionNotice, ''); +}); + test('analysis click does not replace visible channels with an empty successful projection', async () => { const { window } = createDom(); let rebuildClicked = 0;