blomios/Nodecast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: track Laya language batches and normalize labels (#22)

Browse Source 
Report post-projection Laya language detection as a busy analysis phase and normalize Laya language labels before caching and display.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-05 23:42:11 +02:00
parent 45b56d872f
commit a6759b116d
9 changed files with 282 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
server/services/layaService.js
View File

@ -9,6 +9,13 @@ const MAX_TEXT_LENGTH = 4096;
const ALLOWED_MODELS = new Set(['english', 'multilingual', 'typed-decisions']);
const DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000;
const BATCH_TIMEOUT_MS = 2 * 60 * 1000;
const LANGUAGE_CACHE_VERSION = 'v2';
const LANGUAGE_LABELS = new Map([
['fr', 'fr'], ['fr-fr', 'fr'], ['french', 'fr'], ['francais', 'fr'], ['francaise', 'fr'],
['en', 'en'], ['en-gb', 'en'], ['en-us', 'en'], ['english', 'en'], ['anglais', 'en'],
['es', 'es'], ['es-es', 'es'], ['spanish', 'es'], ['espanol', 'es'], ['espagnol', 'es'], ['castellano', 'es'],
['other', 'other'], ['another language', 'other'], ['unknown', 'unknown'], ['undetermined', 'unknown']
]);
class LayaRequestError extends Error {
constructor(message, statusCode = 503) {
@ -120,6 +127,34 @@ function sanitizeIdentityDescriptor(value) {
};
}
function normalizeLanguageLabel(value) {
if (typeof value !== 'string' || !value.trim()) return 'unknown';
const normalized = value
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.trim()
.toLowerCase()
.replace(/_/g, '-');
return LANGUAGE_LABELS.get(normalized) || 'unknown';
}
function parseLanguageAnswer(answer) {
const choice = typeof answer === 'string'
? answer
: answer?.choice ?? answer?.value ?? answer?.label;
if (typeof choice !== 'string' || !choice.trim()) {
throw new LayaRequestError('Laya runtime returned an invalid language answer', 502);
}
const confidenceValue = typeof answer === 'object'
? answer?.answer_confidence ?? answer?.confidence
: 0;
const confidence = Number(confidenceValue);
return {
language: normalizeLanguageLabel(choice),
confidence: Number.isFinite(confidence) ? confidence : 0
};
}
class LayaService {
constructor(options = {}) {
this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000';
@ -291,8 +326,8 @@ class LayaService {
}
}
cacheKey(text, mode, model) {
return crypto.createHash('sha256').update(`${mode}\0${model}\0${text}`).digest('hex');
cacheKey(text, mode, model, version = '') {
return crypto.createHash('sha256').update(`${version}\0${mode}\0${model}\0${text}`).digest('hex');
}
async detectBatch(payload) {
@ -303,7 +338,7 @@ class LayaService {
const misses = [];
payload.items.forEach((item, index) => {
const key = this.cacheKey(item.text, mode, model);
const key = this.cacheKey(item.text, mode, model, LANGUAGE_CACHE_VERSION);
const cached = this.cache ? this.cache.get(key) : null;
if (cached) results[index] = { ...cached, cached: true };
else misses.push({ item, index, key });
@ -328,12 +363,11 @@ class LayaService {
if (!response || !Array.isArray(response.results) || response.results.length !== misses.length) {
throw new LayaRequestError('Laya runtime returned an invalid batch response', 502);
}
const parsed = response.results.map(result => parseLanguageAnswer(result?.answers?.language));
misses.forEach(({ item, index, key }, responseIndex) => {
const answer = response.results[responseIndex]?.answers?.language;
const result = {
id: item.id ?? index,
language: answer?.choice || 'unknown',
confidence: answer?.answer_confidence ?? answer?.confidence ?? 0,
...parsed[responseIndex],
cached: false
};
results[index] = result;
@ -389,6 +423,8 @@ module.exports = {
clearDirectoryContents,
directorySize,
initialDownloadState,
normalizeLanguageLabel,
parseLanguageAnswer,
sanitizeIdentityDescriptor,
validateBatchPayload
};