blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: track Laya language batches and normalize labels (#22)

Browse Source 
Report post-projection Laya language detection as a busy analysis phase and normalize Laya language labels before caching and display.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-05 23:42:11 +02:00
parent 45b56d872f
commit a6759b116d
9 changed files with 282 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
public/js/components/ChannelList.js
View File

@ -344,7 +344,7 @@ class ChannelList {
}
}
async refreshLayaLanguages(mode = window.app?.player?.settings?.languageDetectionMode || 'algo') {
async refreshLayaLanguages(mode = window.app?.player?.settings?.languageDetectionMode || 'algo', options = {}) {
const requestId = ++this._languageDetectionRequest;
if (mode === 'algo') {
const hadWarning = Boolean(this.languageDetectionNotice || this.languageDetectionFallback);
@ -364,6 +364,7 @@ class ChannelList {
);
});
const results = new Map();
const totalBatches = Math.ceil(descriptors.length / 64);
try {
const status = await API.laya.getStatus();
const statusWarning = LayaStatusPresentation.channelListWarning(mode, status);
@ -379,9 +380,15 @@ class ChannelList {
this.languageDetectionNotice = '';
this.languageDetectionFallback = false;
this.render();
options.onProgress?.({ processed: 0, total: totalBatches });
for (let index = 0; index < descriptors.length; index += 64) {
const response = await API.laya.detectLanguages(descriptors.slice(index, index + 64));
(response.results || []).forEach(result => results.set(String(result.id), result));
if (requestId !== this._languageDetectionRequest) return;
options.onProgress?.({
processed: Math.floor(index / 64) + 1,
total: totalBatches
});
}
if (requestId !== this._languageDetectionRequest) return;
if (descriptors.length > 0 && results.size === 0) {
@ -904,7 +911,7 @@ class ChannelList {
/**
* Load channels from selected source
*/
async loadChannels() {
async loadChannels({ refreshLanguages = true } = {}) {
if (this.isLoading) return;
this.isLoading = true;
this.currentRenderId = null; // Reset render tracking
@ -921,7 +928,7 @@ class ChannelList {
if (!sourceValue) {
// Load from all sources
await this.loadAllChannels();
await this.loadAllChannels({ refreshLanguages });
this.isLoading = false;
return;
}
@ -933,7 +940,7 @@ class ChannelList {
if (await this.loadProjectedChannels(parseInt(id, 10))) {
this.render();
void this.refreshLayaLanguages();
if (refreshLanguages) void this.refreshLayaLanguages();
return;
}
@ -950,7 +957,7 @@ class ChannelList {
]);
this.render();
void this.refreshLayaLanguages();
if (refreshLanguages) void this.refreshLayaLanguages();
} catch (err) {
console.error('Error loading channels:', err);
if (previousState.channels.length > 0) {
@ -970,7 +977,7 @@ class ChannelList {
/**
* Load channels from all enabled sources
*/
async loadAllChannels() {
async loadAllChannels({ refreshLanguages = true } = {}) {
const previousState = {
channels: this.channels,
groups: this.groups,
@ -985,7 +992,7 @@ class ChannelList {
if (await this.loadProjectedChannels()) {
this.render();
void this.refreshLayaLanguages();
if (refreshLanguages) void this.refreshLayaLanguages();
return;
}
@ -1006,7 +1013,7 @@ class ChannelList {
this.loadFavorites()
]);
this.render();
void this.refreshLayaLanguages();
if (refreshLanguages) void this.refreshLayaLanguages();
} catch (err) {
console.error('Error loading all channels:', err);
if (previousState.channels.length > 0) {

25
public/js/pages/Settings.js
View File

@ -197,11 +197,32 @@ class SettingsPage {
clearInterval(this.liveAnalysisPollTimer);
this.liveAnalysisPollTimer = null;
if (presentation.complete) {
await this.app.channelList?.loadChannels();
const mode = this.app.player?.settings?.languageDetectionMode || 'algo';
await this.app.channelList?.loadChannels({
refreshLanguages: mode !== 'laya' && mode !== 'hybrid'
});
if (mode === 'laya' || mode === 'hybrid') {
await this.app.channelList?.refreshLayaLanguages(mode);
this.liveAnalysisStatus = {
status: 'building',
phase: 'laya_language',
languageProcessed: null,
languageTotal: null
};
this.renderLiveAnalysisStatus(this.liveAnalysisStatus);
await this.app.channelList?.refreshLayaLanguages(mode, {
onProgress: ({ processed, total }) => {
this.liveAnalysisStatus = {
status: 'building',
phase: 'laya_language',
languageProcessed: processed,
languageTotal: total
};
this.renderLiveAnalysisStatus(this.liveAnalysisStatus);
}
});
}
this.liveAnalysisStatus = status;
this.renderLiveAnalysisStatus(status);
}
}
} finally {

16
public/js/services/channelLanguageFusion.js
View File

@ -10,9 +10,19 @@
return Math.max(0, Math.min(100, Math.round(number <= 1 ? number * 100 : number)));
}
function normalizeLanguageCode(value) {
const normalized = String(value || '').trim().toLowerCase();
const aliases = {
fr: 'fr', french: 'fr', français: 'fr', francais: 'fr',
en: 'en', english: 'en', anglais: 'en',
es: 'es', spanish: 'es', español: 'es', espanol: 'es'
};
return aliases[normalized] || 'unknown';
}
function layaDetection(result) {
const rawCode = result?.language?.code || result?.language || 'unknown';
const code = ['fr', 'en', 'es'].includes(rawCode) ? rawCode : 'unknown';
const rawCode = result?.language?.code || result?.language?.label || result?.language || 'unknown';
const code = normalizeLanguageCode(rawCode);
const confidence = normalizeConfidence(result?.language?.confidence ?? result?.confidence);
return {
language: {
@ -79,5 +89,5 @@
};
}
return { applyLanguageMode, buildLayaDescriptor, layaDetection, mergeHybrid, normalizeConfidence };
return { applyLanguageMode, buildLayaDescriptor, layaDetection, mergeHybrid, normalizeConfidence, normalizeLanguageCode };
});

9
public/js/services/liveAnalysisPresentation.js
View File

@ -14,6 +14,13 @@
prefix: 'AI batch'
};
}
if (phase === 'laya_language') {
return {
processed: status.languageProcessed,
total: status.languageTotal,
prefix: 'Language batch'
};
}
return { processed: status.processed, total: status.total, prefix: '', suffix: ' channels' };
}
@ -37,6 +44,8 @@
const statusLabel = busy
? phase === 'laya_identity'
? 'Analyzing with local AI…'
: phase === 'laya_language'
? 'Detecting stream languages…'
: phase === 'projecting'
? 'Building channel catalog…'
: phase === 'writing'

48
server/services/layaService.js
View File

@ -9,6 +9,13 @@ const MAX_TEXT_LENGTH = 4096;
const ALLOWED_MODELS = new Set(['english', 'multilingual', 'typed-decisions']);
const DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000;
const BATCH_TIMEOUT_MS = 2 * 60 * 1000;
const LANGUAGE_CACHE_VERSION = 'v2';
const LANGUAGE_LABELS = new Map([
['fr', 'fr'], ['fr-fr', 'fr'], ['french', 'fr'], ['francais', 'fr'], ['francaise', 'fr'],
['en', 'en'], ['en-gb', 'en'], ['en-us', 'en'], ['english', 'en'], ['anglais', 'en'],
['es', 'es'], ['es-es', 'es'], ['spanish', 'es'], ['espanol', 'es'], ['espagnol', 'es'], ['castellano', 'es'],
['other', 'other'], ['another language', 'other'], ['unknown', 'unknown'], ['undetermined', 'unknown']
]);
class LayaRequestError extends Error {
constructor(message, statusCode = 503) {
@ -120,6 +127,34 @@ function sanitizeIdentityDescriptor(value) {
};
}
function normalizeLanguageLabel(value) {
if (typeof value !== 'string' || !value.trim()) return 'unknown';
const normalized = value
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.trim()
.toLowerCase()
.replace(/_/g, '-');
return LANGUAGE_LABELS.get(normalized) || 'unknown';
}
function parseLanguageAnswer(answer) {
const choice = typeof answer === 'string'
? answer
: answer?.choice ?? answer?.value ?? answer?.label;
if (typeof choice !== 'string' || !choice.trim()) {
throw new LayaRequestError('Laya runtime returned an invalid language answer', 502);
}
const confidenceValue = typeof answer === 'object'
? answer?.answer_confidence ?? answer?.confidence
: 0;
const confidence = Number(confidenceValue);
return {
language: normalizeLanguageLabel(choice),
confidence: Number.isFinite(confidence) ? confidence : 0
};
}
class LayaService {
constructor(options = {}) {
this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000';
@ -291,8 +326,8 @@ class LayaService {
}
}
cacheKey(text, mode, model) {
return crypto.createHash('sha256').update(`${mode}\0${model}\0${text}`).digest('hex');
cacheKey(text, mode, model, version = '') {
return crypto.createHash('sha256').update(`${version}\0${mode}\0${model}\0${text}`).digest('hex');
}
async detectBatch(payload) {
@ -303,7 +338,7 @@ class LayaService {
const misses = [];
payload.items.forEach((item, index) => {
const key = this.cacheKey(item.text, mode, model);
const key = this.cacheKey(item.text, mode, model, LANGUAGE_CACHE_VERSION);
const cached = this.cache ? this.cache.get(key) : null;
if (cached) results[index] = { ...cached, cached: true };
else misses.push({ item, index, key });
@ -328,12 +363,11 @@ class LayaService {
if (!response || !Array.isArray(response.results) || response.results.length !== misses.length) {
throw new LayaRequestError('Laya runtime returned an invalid batch response', 502);
}
const parsed = response.results.map(result => parseLanguageAnswer(result?.answers?.language));
misses.forEach(({ item, index, key }, responseIndex) => {
const answer = response.results[responseIndex]?.answers?.language;
const result = {
id: item.id ?? index,
language: answer?.choice || 'unknown',
confidence: answer?.answer_confidence ?? answer?.confidence ?? 0,
...parsed[responseIndex],
cached: false
};
results[index] = result;
@ -389,6 +423,8 @@ module.exports = {
clearDirectoryContents,
directorySize,
initialDownloadState,
normalizeLanguageLabel,
parseLanguageAnswer,
sanitizeIdentityDescriptor,
validateBatchPayload
};

6
test/channelLanguageFusion.test.js
View File

@ -2,6 +2,12 @@ const test = require('node:test');
const assert = require('node:assert/strict');
const fusion = require('../public/js/services/channelLanguageFusion.js');
test('normalizes common Laya language labels to supported codes', () => {
assert.equal(fusion.normalizeLanguageCode('French'), 'fr');
assert.equal(fusion.normalizeLanguageCode('English'), 'en');
assert.equal(fusion.normalizeLanguageCode('ESPAÑOL'), 'es');
});
const algo = {
language: { code: 'fr', label: 'French', confidence: 70, status: 'medium' },
languageScores: { fr: 70, en: 30 },

43
test/layaService.test.js
View File

@ -152,8 +152,8 @@ test('language batch sends the Laya contract and parses language confidence with
ok: true,
json: async () => ({
results: [
{ answers: { language: { choice: 'fr', answer_confidence: 0.93 } } },
{ answers: { language: { choice: 'en', confidence: 0.81 } } }
{ answers: { language: { choice: 'French', answer_confidence: 0.93 } } },
{ answers: { language: { choice: 'English', confidence: 0.81 } } }
]
})
};
@ -181,6 +181,45 @@ test('language batch sends the Laya contract and parses language confidence with
});
});
test('language batch preserves explicit other and maps an unfamiliar label to unknown', async () => {
const service = new LayaService({
cache: null,
fetch: async () => ({
ok: true,
json: async () => ({
results: [
{ answers: { language: { choice: 'Other', confidence: 0.72 } } },
{ answers: { language: { choice: 'Klingon', confidence: 0.99 } } }
]
})
})
});
const response = await service.detectBatch({
items: [{ id: 'other', text: 'Autre' }, { id: 'strange', text: 'Qapla' }]
});
assert.deepEqual(response.results, [
{ id: 'other', language: 'other', confidence: 0.72, cached: false },
{ id: 'strange', language: 'unknown', confidence: 0.99, cached: false }
]);
});
test('language batch rejects a successful HTTP response without a usable language answer', async () => {
const service = new LayaService({
cache: null,
fetch: async () => ({
ok: true,
json: async () => ({ results: [{ answers: { language: { confidence: 0.9 } } }] })
})
});
await assert.rejects(
service.detectBatch({ items: [{ id: 'missing', text: 'Missing language' }] }),
error => error instanceof LayaRequestError && error.statusCode === 502
);
});
test('identity hints send only sanitized channel metadata to Laya', async () => {
let requestBody;
const service = new LayaService({

13
test/liveAnalysisPresentation.test.js
View File

@ -26,6 +26,19 @@ test('presents Laya identity progress using AI batch counters', () => {
assert.equal(view.progressText, 'AI batch 3 of 8 · 38%');
});
test('presents local Laya language batch progress after projection', () => {
const view = present({
status: 'building',
phase: 'laya_language',
languageProcessed: 5,
languageTotal: 10
});
assert.equal(view.statusLabel, 'Detecting stream languages…');
assert.equal(view.progressIndeterminate, false);
assert.equal(view.progressPercent, 50);
assert.equal(view.progressText, 'Language batch 5 of 10 · 50%');
});
test('presents indeterminate progress when totals are unavailable', () => {
const view = present({ status: 'queued', processed: null, total: null });
assert.equal(view.busy, true);

120
test/liveAnalysisUserFlowDom.test.js
View File

@ -92,6 +92,18 @@ function projectedChannels() {
};
}
function waitFor(condition, message) {
return new Promise((resolve, reject) => {
const started = Date.now();
const poll = () => {
if (condition()) return resolve();
if (Date.now() - started > 1500) return reject(new Error(message));
setTimeout(poll, 5);
};
poll();
});
}
test('Settings renders Laya batch progress and removes misleading values when its total is unknown', () => {
const { window } = createDom();
const settingsPage = Object.create(window.SettingsPage.prototype);
@ -129,6 +141,67 @@ test('Settings renders Laya batch progress and removes misleading values when it
assert.equal(bar.style.width, '');
});
test('analysis remains visibly busy while Laya language batches run after projection completes', async () => {
const { window } = createDom();
let rebuildClicked = false;
let languageRefreshStarted = false;
let releaseLanguageRefresh;
const languageRefresh = new Promise(resolve => { releaseLanguageRefresh = resolve; });
window.API = {
live: {
getProjectionStatus: async () => rebuildClicked
? { status: 'success', phase: 'complete', processed: 128, total: 128, projectedCount: 64 }
: { status: 'idle', processed: 0, total: 128 },
rebuildProjection: async () => {
rebuildClicked = true;
return {
projection: {
status: 'building',
phase: 'laya_identity',
layaProcessed: 4,
layaTotal: 4,
processed: 0,
total: 128
}
};
}
},
laya: {
getStatus: async () => ({ modelPresent: true, runtimeAvailable: true })
}
};
const app = {
player: {
settings: { languageDetectionMode: 'laya' },
saveSettings() {}
},
channelList: {
loadChannels: async () => {},
refreshLayaLanguages: async () => {
languageRefreshStarted = true;
await languageRefresh;
}
}
};
window.app = app;
const settingsPage = Object.create(window.SettingsPage.prototype);
settingsPage.app = app;
settingsPage.layaStatus = { modelPresent: true, runtimeAvailable: true };
settingsPage.initLiveAnalysis();
await Promise.resolve();
window.document.getElementById('live-analysis-button').click();
try {
await waitFor(() => languageRefreshStarted, 'Expected Laya language refresh to start after projection completion');
assert.notEqual(window.document.getElementById('live-analysis-status').textContent, 'Analysis complete');
assert.equal(window.document.getElementById('live-analysis-progress').hidden, false);
} finally {
releaseLanguageRefresh();
window.clearInterval(settingsPage.liveAnalysisPollTimer);
}
});
test('user analysis click keeps visible channels after Local AI language failure', async () => {
const { window } = createDom();
let rebuildClicked = 0;
@ -348,6 +421,53 @@ test('successful Laya language batch updates visible variant languages without S
assert.equal(window.document.querySelector('.laya-language-notice'), null);
});
test('usable Laya language labels do not leave visible variants as unknown', async () => {
const { window } = createDom();
window.API = {
live: {
getChannels: async () => projectedChannels()
},
laya: {
getStatus: async () => ({ modelPresent: true, runtimeAvailable: true }),
detectLanguages: async items => ({
runtimeAvailable: true,
model: 'multilingual',
results: items.map(item => ({
id: item.id,
language: item.id.endsWith('fr-sd') ? 'French' : 'English',
confidence: 0.91
}))
})
}
};
const app = {
player: {
settings: { liveChannelListMode: 'grouped', languageDetectionMode: 'laya' },
saveSettings() {}
},
epgGuide: null,
navigateTo() {},
pages: { settings: { switchTab() {} } }
};
window.app = app;
const channelList = new window.ChannelList();
channelList.sources = [{ id: 1, type: 'm3u', name: 'Fixture Provider' }];
app.channelList = channelList;
await channelList.loadChannels();
await channelList.refreshLayaLanguages('laya');
channelList.expandAll();
const badges = [...window.document.querySelectorAll('.variant-language')].map(item => item.textContent);
window.clearInterval(channelList._epgRefreshTimer);
assert.deepEqual([...new Set(badges)].sort(), ['EN 91%', 'FR 91%']);
assert.equal(channelList.languageDetectionFallback, false);
assert.equal(channelList.languageDetectionNotice, '');
});
test('analysis click does not replace visible channels with an empty successful projection', async () => {
const { window } = createDom();
let rebuildClicked = 0;