feat: add optional Laya engine mode for grouping and language detection (#12)
Add a languageDetectionMode setting (algo | laya | laya+algo) exposed in Settings with three engine choices. When Laya is enabled, a laya service (Dockerfile.laya, optional compose profile, models stored outside the repo under ./data/models/laya) can be started, with model download and deletion from the UI via /laya routes to manage disk space. Add a channelLanguageFusion service combining Laya results with the existing algorithm, and wire detection mode through grouping, channel list and player. Cover engine selection, fusion and laya service behavior with tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
23
Dockerfile.laya
Normal file
23
Dockerfile.laya
Normal file
@ -0,0 +1,23 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ARG TORCH_VERSION=2.14.1
|
||||
|
||||
# Install the CPU wheel explicitly before Laya so pip does not select the much
|
||||
# larger CUDA build. Model weights remain runtime-only in the mounted cache.
|
||||
RUN python -m pip install --no-cache-dir \
|
||||
"torch==${TORCH_VERSION}" \
|
||||
--index-url https://download.pytorch.org/whl/cpu \
|
||||
&& python -m pip install --no-cache-dir "laya[serve]>=0.3.22,<0.4" \
|
||||
&& python -m pip check \
|
||||
&& python -c "import torch; assert torch.version.cuda is None, torch.version.cuda"
|
||||
|
||||
ENV HF_HOME=/models \
|
||||
LAYA_HOST=0.0.0.0 \
|
||||
LAYA_PORT=8000 \
|
||||
LAYA_PRELOAD=0 \
|
||||
LAYA_DEVICE=cpu
|
||||
|
||||
VOLUME ["/models"]
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["laya-serve"]
|
||||
@ -13,5 +13,20 @@ services:
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- PORT=3000 # Internal container port
|
||||
- LAYA_BASE_URL=http://laya:8000
|
||||
devices:
|
||||
- /dev/dri:/dev/dri # Required for VAAPI/QuickSync/AMF (Linux)
|
||||
|
||||
laya:
|
||||
profiles: ["laya"]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.laya
|
||||
volumes:
|
||||
- ./data/models/laya:/models
|
||||
environment:
|
||||
- HF_HOME=/models
|
||||
- LAYA_PRELOAD=0
|
||||
- LAYA_HOST=0.0.0.0
|
||||
- LAYA_PORT=8000
|
||||
restart: unless-stopped
|
||||
|
||||
@ -725,6 +725,71 @@ h4 {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.channel-matching-settings {
|
||||
margin-top: var(--space-lg);
|
||||
padding-top: var(--space-md);
|
||||
border-top: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.matching-mode-options {
|
||||
display: grid;
|
||||
gap: var(--space-sm);
|
||||
margin: var(--space-md) 0;
|
||||
}
|
||||
|
||||
.matching-mode-card {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: var(--space-sm);
|
||||
padding: var(--space-md);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-sm);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.matching-mode-card:has(input:checked) {
|
||||
border-color: var(--color-accent);
|
||||
background: var(--color-accent-dim);
|
||||
}
|
||||
|
||||
.matching-mode-card span,
|
||||
.matching-mode-card small {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.matching-mode-card small {
|
||||
margin-top: 4px;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.laya-model-panel {
|
||||
padding: var(--space-md);
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--color-bg-tertiary);
|
||||
}
|
||||
|
||||
.laya-model-heading,
|
||||
.laya-model-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-sm);
|
||||
}
|
||||
|
||||
.laya-model-actions {
|
||||
justify-content: flex-start;
|
||||
margin: var(--space-md) 0;
|
||||
}
|
||||
|
||||
.laya-language-notice {
|
||||
margin: var(--space-sm) var(--space-md);
|
||||
padding: var(--space-sm);
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--color-bg-tertiary);
|
||||
color: var(--color-text-muted);
|
||||
font-size: .75rem;
|
||||
}
|
||||
|
||||
.variant-name {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
<title>nodecast-tv | IPTV Player</title>
|
||||
<meta name="description" content="Stream live TV, movies, and series with EPG support">
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
|
||||
<link rel="stylesheet" href="css/main.css?v=2">
|
||||
<link rel="stylesheet" href="css/main.css?v=3">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Outfit:wght@800&display=swap"
|
||||
@ -466,6 +466,36 @@
|
||||
<span class="setting-hint">Turn off to show all channels in a single list. Stream variants remain grouped by channel.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="channel-matching-settings">
|
||||
<h4>Channel matching</h4>
|
||||
<p class="setting-hint">Choose how Nodecast TV identifies matching streams and languages.</p>
|
||||
<div class="matching-mode-options">
|
||||
<label class="matching-mode-card">
|
||||
<input type="radio" name="language-detection-mode" value="algo" checked>
|
||||
<span><strong>Standard (Recommended)</strong><small>Use the built-in matching system. Fast and no download required.</small></span>
|
||||
</label>
|
||||
<label class="matching-mode-card">
|
||||
<input type="radio" name="language-detection-mode" value="laya">
|
||||
<span><strong>Local AI only</strong><small>Use the downloaded AI model instead of the built-in matching system.</small></span>
|
||||
</label>
|
||||
<label class="matching-mode-card">
|
||||
<input type="radio" name="language-detection-mode" value="hybrid">
|
||||
<span><strong>Enhanced</strong><small>Combine the local AI model with the built-in matching system for more evidence.</small></span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="laya-model-panel" id="laya-model-panel">
|
||||
<div class="laya-model-heading">
|
||||
<div><strong>Local AI model · Laya</strong><div id="laya-model-status" class="setting-hint" role="status">Checking status…</div></div>
|
||||
<span id="laya-model-size" class="setting-hint"></span>
|
||||
</div>
|
||||
<div id="laya-model-message" class="setting-hint"></div>
|
||||
<div class="laya-model-actions">
|
||||
<button type="button" class="btn btn-primary" id="laya-download-model">Download model</button>
|
||||
<button type="button" class="btn btn-ghost" id="laya-remove-model">Remove downloaded model</button>
|
||||
</div>
|
||||
<p class="setting-hint">The model runs on this server. Channel names and metadata are not sent to an AI service.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="settings-section">
|
||||
@ -1137,11 +1167,12 @@
|
||||
|
||||
<!-- JavaScript -->
|
||||
<script src="/js/icons.js"></script>
|
||||
<script src="/js/api.js?v=2"></script>
|
||||
<script src="/js/components/VideoPlayer.js?v=4"></script>
|
||||
<script src="/js/api.js?v=3"></script>
|
||||
<script src="/js/components/VideoPlayer.js?v=5"></script>
|
||||
<script src="/js/services/channelLanguageDetection.js?v=1"></script>
|
||||
<script src="/js/services/channelVariantGrouping.js?v=3"></script>
|
||||
<script src="/js/components/ChannelList.js?v=6"></script>
|
||||
<script src="/js/services/channelLanguageFusion.js?v=1"></script>
|
||||
<script src="/js/services/channelVariantGrouping.js?v=4"></script>
|
||||
<script src="/js/components/ChannelList.js?v=7"></script>
|
||||
<script src="/js/components/SourceManager.js?v=2"></script>
|
||||
<script src="/js/components/EpgGuide.js?v=2"></script>
|
||||
<script src="/js/pages/HomePage.js?v=2"></script>
|
||||
@ -1149,7 +1180,7 @@
|
||||
<script src="/js/pages/Guide.js?v=2"></script>
|
||||
<script src="/js/pages/MoviesPage.js?v=4"></script>
|
||||
<script src="/js/pages/SeriesPage.js?v=1"></script>
|
||||
<script src="/js/pages/Settings.js?v=3"></script>
|
||||
<script src="/js/pages/Settings.js?v=4"></script>
|
||||
<script src="/js/pages/WatchPage.js?v=1"></script>
|
||||
<script src="/js/app.js?v=4"></script>
|
||||
<script>
|
||||
|
||||
@ -95,6 +95,13 @@ const API = {
|
||||
API.request('GET', `/favorites/check?sourceId=${sourceId}&itemId=${itemId}&itemType=${itemType}`)
|
||||
},
|
||||
|
||||
laya: {
|
||||
getStatus: () => API.request('GET', '/laya/status'),
|
||||
downloadModel: (model = 'multilingual') => API.request('POST', '/laya/model/download', { model }),
|
||||
removeModel: () => API.request('DELETE', '/laya/model'),
|
||||
detectLanguages: (items, model = 'multilingual') => API.request('POST', '/laya/language/batch', { items, model })
|
||||
},
|
||||
|
||||
// Proxy
|
||||
proxy: {
|
||||
// Xtream
|
||||
|
||||
@ -31,6 +31,9 @@ class ChannelList {
|
||||
this.channelLanguagePreferencesKey = 'nodecast_tv_channel_language_preferences';
|
||||
this.preferredLanguage = this.loadPreferredLanguage();
|
||||
this.channelLanguagePreferences = this.loadChannelLanguagePreferences();
|
||||
this.layaLanguageResults = new Map();
|
||||
this.languageDetectionNotice = '';
|
||||
this._languageDetectionRequest = 0;
|
||||
|
||||
this.loadCollapsedState();
|
||||
this.init();
|
||||
@ -328,6 +331,52 @@ class ChannelList {
|
||||
if (this.channels.length > 0) this.render();
|
||||
}
|
||||
|
||||
setLanguageDetectionMode(mode) {
|
||||
const safeMode = ['algo', 'laya', 'hybrid'].includes(mode) ? mode : 'algo';
|
||||
this.layaLanguageResults.clear();
|
||||
this.languageDetectionNotice = '';
|
||||
if (this.channels.length > 0) {
|
||||
this.render();
|
||||
void this.refreshLayaLanguages(safeMode);
|
||||
}
|
||||
}
|
||||
|
||||
async refreshLayaLanguages(mode = window.app?.player?.settings?.languageDetectionMode || 'algo') {
|
||||
const requestId = ++this._languageDetectionRequest;
|
||||
if (mode === 'algo') {
|
||||
this.layaLanguageResults.clear();
|
||||
this.languageDetectionNotice = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const descriptors = this.channels.map(variant => {
|
||||
const algo = ChannelLanguageDetection.detectLanguage(variant, { sourceName: variant.sourceName });
|
||||
return ChannelLanguageFusion.buildLayaDescriptor(
|
||||
variant,
|
||||
ChannelVariantGrouping.getVariantKey(variant),
|
||||
algo
|
||||
);
|
||||
});
|
||||
const results = new Map();
|
||||
try {
|
||||
for (let index = 0; index < descriptors.length; index += 64) {
|
||||
const response = await API.laya.detectLanguages(descriptors.slice(index, index + 64));
|
||||
(response.results || []).forEach(result => results.set(String(result.id), result));
|
||||
}
|
||||
if (requestId !== this._languageDetectionRequest) return;
|
||||
this.layaLanguageResults = results;
|
||||
this.languageDetectionNotice = '';
|
||||
} catch (err) {
|
||||
if (requestId !== this._languageDetectionRequest) return;
|
||||
this.layaLanguageResults.clear();
|
||||
this.languageDetectionNotice = mode === 'hybrid'
|
||||
? 'Local AI is unavailable; using standard channel matching.'
|
||||
: 'Local AI is unavailable. Download or start the model in Settings.';
|
||||
console.warn('[ChannelList] Local AI language detection unavailable:', err.message);
|
||||
}
|
||||
this.render();
|
||||
}
|
||||
|
||||
escapeHtml(text) {
|
||||
if (!text) return '';
|
||||
return text
|
||||
@ -356,6 +405,8 @@ class ChannelList {
|
||||
selectedVariants: this.selectedVariants,
|
||||
preferredLanguage: this.preferredLanguage,
|
||||
languagePreferences: this.channelLanguagePreferences,
|
||||
languageMode: window.app?.player?.settings?.languageDetectionMode || 'algo',
|
||||
languageResults: this.layaLanguageResults,
|
||||
isFavorite: channel => this.isFavorite(channel.sourceId, channel.id),
|
||||
isHidden: channel => this.isHidden('channel', channel.sourceId, channel.streamId || channel.id)
|
||||
});
|
||||
@ -364,6 +415,13 @@ class ChannelList {
|
||||
this.currentBatch = 0;
|
||||
this.batchSize = 100; // Number of groups to render per batch (increased to handle many hidden groups)
|
||||
this.container.innerHTML = ''; // Clear container
|
||||
if (this.languageDetectionNotice) {
|
||||
const notice = document.createElement('div');
|
||||
notice.className = 'laya-language-notice';
|
||||
notice.setAttribute('role', 'status');
|
||||
notice.textContent = this.languageDetectionNotice;
|
||||
this.container.appendChild(notice);
|
||||
}
|
||||
|
||||
// Filter and Group channels
|
||||
const groupedChannels = {};
|
||||
@ -837,6 +895,7 @@ class ChannelList {
|
||||
]);
|
||||
|
||||
this.render();
|
||||
void this.refreshLayaLanguages();
|
||||
} catch (err) {
|
||||
console.error('Error loading channels:', err);
|
||||
this.container.innerHTML = `<div class="empty-state"><p>Error loading channels</p><p class="hint">${err.message}</p></div>`;
|
||||
@ -872,6 +931,7 @@ class ChannelList {
|
||||
this.loadFavorites()
|
||||
]);
|
||||
this.render();
|
||||
void this.refreshLayaLanguages();
|
||||
} catch (err) {
|
||||
console.error('Error loading all channels:', err);
|
||||
}
|
||||
|
||||
@ -58,6 +58,7 @@ class VideoPlayer {
|
||||
lastVolume: 80,
|
||||
autoPlayNextEpisode: false,
|
||||
liveChannelListMode: 'grouped',
|
||||
languageDetectionMode: 'algo',
|
||||
forceProxy: false,
|
||||
forceTranscode: false,
|
||||
forceRemux: false,
|
||||
|
||||
@ -86,6 +86,8 @@ class SettingsPage {
|
||||
this.app.player.saveSettings();
|
||||
});
|
||||
|
||||
this.initLayaSettings();
|
||||
|
||||
// EPG refresh interval
|
||||
const epgRefreshSelect = document.getElementById('epg-refresh-interval');
|
||||
if (epgRefreshSelect && this.app.player?.settings) {
|
||||
@ -103,6 +105,95 @@ class SettingsPage {
|
||||
this.updateEpgLastRefreshed();
|
||||
}
|
||||
|
||||
initLayaSettings() {
|
||||
const radios = document.querySelectorAll('input[name="language-detection-mode"]');
|
||||
const currentMode = this.app.player?.settings?.languageDetectionMode || 'algo';
|
||||
radios.forEach(radio => {
|
||||
radio.checked = radio.value === currentMode;
|
||||
radio.addEventListener('change', () => {
|
||||
if (!radio.checked) return;
|
||||
this.app.player.settings.languageDetectionMode = radio.value;
|
||||
this.app.player.saveSettings();
|
||||
this.app.channelList.setLanguageDetectionMode(radio.value);
|
||||
this.renderLayaStatus(this.layaStatus);
|
||||
});
|
||||
});
|
||||
|
||||
document.getElementById('laya-download-model')?.addEventListener('click', () => this.runLayaModelAction('download'));
|
||||
document.getElementById('laya-remove-model')?.addEventListener('click', () => {
|
||||
if (confirm('Remove local AI model?\n\nYour sources, favorites, and hidden channels will not be deleted.')) {
|
||||
this.runLayaModelAction('remove');
|
||||
}
|
||||
});
|
||||
void this.loadLayaStatus();
|
||||
}
|
||||
|
||||
async loadLayaStatus() {
|
||||
try {
|
||||
this.layaStatus = await API.laya.getStatus();
|
||||
} catch (err) {
|
||||
this.layaStatus = { runtimeAvailable: false, modelPresent: false, error: err.message };
|
||||
}
|
||||
this.renderLayaStatus(this.layaStatus);
|
||||
return this.layaStatus;
|
||||
}
|
||||
|
||||
renderLayaStatus(status = {}) {
|
||||
const statusElement = document.getElementById('laya-model-status');
|
||||
const sizeElement = document.getElementById('laya-model-size');
|
||||
const messageElement = document.getElementById('laya-model-message');
|
||||
const downloadButton = document.getElementById('laya-download-model');
|
||||
const removeButton = document.getElementById('laya-remove-model');
|
||||
if (!statusElement) return;
|
||||
|
||||
const modelPresent = status.modelPresent === true;
|
||||
const runtimeAvailable = status.runtimeAvailable === true;
|
||||
statusElement.textContent = modelPresent
|
||||
? (runtimeAvailable ? 'Model ready' : 'Model downloaded · runtime unavailable')
|
||||
: (runtimeAvailable ? 'Model not downloaded' : 'Local AI runtime unavailable');
|
||||
const size = status.modelSizeBytes || status.sizeBytes || status.runtime?.modelSizeBytes;
|
||||
sizeElement.textContent = size ? this.formatBytes(size) : '';
|
||||
downloadButton.hidden = modelPresent;
|
||||
removeButton.hidden = !modelPresent;
|
||||
downloadButton.disabled = Boolean(this.layaOperationPending);
|
||||
removeButton.disabled = Boolean(this.layaOperationPending);
|
||||
|
||||
const mode = this.app.player?.settings?.languageDetectionMode || 'algo';
|
||||
if (this.layaOperationPending) messageElement.textContent = this.layaOperationPending;
|
||||
else if (mode !== 'algo' && (!modelPresent || !runtimeAvailable)) {
|
||||
messageElement.textContent = 'Local AI is not ready. Download the model and ensure the runtime is running. Live TV will remain available.';
|
||||
} else messageElement.textContent = status.error || '';
|
||||
}
|
||||
|
||||
formatBytes(bytes) {
|
||||
const value = Number(bytes);
|
||||
if (!Number.isFinite(value) || value <= 0) return '';
|
||||
const units = ['B', 'KB', 'MB', 'GB'];
|
||||
const index = Math.min(Math.floor(Math.log(value) / Math.log(1024)), units.length - 1);
|
||||
return `${(value / (1024 ** index)).toFixed(index > 1 ? 1 : 0)} ${units[index]}`;
|
||||
}
|
||||
|
||||
async runLayaModelAction(action) {
|
||||
if (this.layaOperationPending) return;
|
||||
this.layaOperationPending = action === 'download' ? 'Downloading model…' : 'Removing model…';
|
||||
this.renderLayaStatus(this.layaStatus);
|
||||
const poll = setInterval(() => void this.loadLayaStatus(), 2000);
|
||||
try {
|
||||
this.layaStatus = action === 'download'
|
||||
? await API.laya.downloadModel('multilingual')
|
||||
: await API.laya.removeModel();
|
||||
} catch (err) {
|
||||
this.layaStatus = { ...(this.layaStatus || {}), error: err.message };
|
||||
} finally {
|
||||
clearInterval(poll);
|
||||
this.layaOperationPending = '';
|
||||
await this.loadLayaStatus();
|
||||
if (this.app.player.settings.languageDetectionMode !== 'algo') {
|
||||
this.app.channelList.setLanguageDetectionMode(this.app.player.settings.languageDetectionMode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async initTranscodingSettings() {
|
||||
// Encoder settings
|
||||
const hwEncoderSelect = document.getElementById('setting-hw-encoder');
|
||||
@ -595,6 +686,9 @@ class SettingsPage {
|
||||
if (rememberVolumeToggle) rememberVolumeToggle.checked = s.rememberVolume;
|
||||
if (autoPlayNextToggle) autoPlayNextToggle.checked = s.autoPlayNextEpisode;
|
||||
if (groupLiveChannelsToggle) groupLiveChannelsToggle.checked = s.liveChannelListMode !== 'flat';
|
||||
document.querySelectorAll('input[name="language-detection-mode"]').forEach(radio => {
|
||||
radio.checked = radio.value === (s.languageDetectionMode || 'algo');
|
||||
});
|
||||
if (forceProxyToggle) forceProxyToggle.checked = s.forceProxy || false;
|
||||
if (forceTranscodeToggle) forceTranscodeToggle.checked = s.forceTranscode || false;
|
||||
if (forceRemuxToggle) forceRemuxToggle.checked = s.forceRemux || false;
|
||||
@ -617,6 +711,7 @@ class SettingsPage {
|
||||
|
||||
// Update EPG last refreshed display
|
||||
this.updateEpgLastRefreshed();
|
||||
void this.loadLayaStatus();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
83
public/js/services/channelLanguageFusion.js
Normal file
83
public/js/services/channelLanguageFusion.js
Normal file
@ -0,0 +1,83 @@
|
||||
(function (root, factory) {
|
||||
const service = factory();
|
||||
if (typeof module === 'object' && module.exports) module.exports = service;
|
||||
if (root) root.ChannelLanguageFusion = service;
|
||||
})(typeof globalThis !== 'undefined' ? globalThis : this, function () {
|
||||
const LABELS = { fr: 'French', en: 'English', es: 'Spanish', unknown: 'Unknown' };
|
||||
|
||||
function normalizeConfidence(value) {
|
||||
const number = Number(value) || 0;
|
||||
return Math.max(0, Math.min(100, Math.round(number <= 1 ? number * 100 : number)));
|
||||
}
|
||||
|
||||
function layaDetection(result) {
|
||||
const rawCode = result?.language?.code || result?.language || 'unknown';
|
||||
const code = ['fr', 'en', 'es'].includes(rawCode) ? rawCode : 'unknown';
|
||||
const confidence = normalizeConfidence(result?.language?.confidence ?? result?.confidence);
|
||||
return {
|
||||
language: {
|
||||
code,
|
||||
label: LABELS[code],
|
||||
confidence: code === 'unknown' ? 0 : confidence,
|
||||
status: code === 'unknown' ? 'unknown' : confidence >= 75 ? 'high' : confidence >= 60 ? 'medium' : 'low'
|
||||
},
|
||||
languageScores: code === 'unknown' ? {} : { [code]: confidence },
|
||||
languageEvidence: code === 'unknown' ? [] : [{ code, weight: confidence, type: 'laya', value: 'Local AI model' }]
|
||||
};
|
||||
}
|
||||
|
||||
function mergeHybrid(algo, result) {
|
||||
if (!result) return algo;
|
||||
const laya = layaDetection(result);
|
||||
if (laya.language.code === 'unknown') return {
|
||||
...algo,
|
||||
languageEvidence: [...algo.languageEvidence, ...laya.languageEvidence]
|
||||
};
|
||||
const combined = { ...algo.languageScores };
|
||||
combined[laya.language.code] = (combined[laya.language.code] || 0) + laya.language.confidence;
|
||||
const total = Object.values(combined).reduce((sum, value) => sum + value, 0);
|
||||
const scores = Object.fromEntries(Object.entries(combined).map(([code, value]) => [code, Math.round(value / total * 100)]));
|
||||
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
|
||||
const [code, confidence] = ranked[0];
|
||||
const margin = confidence - (ranked[1]?.[1] || 0);
|
||||
const certain = confidence >= 45 && margin >= 10;
|
||||
return {
|
||||
language: certain ? {
|
||||
code,
|
||||
label: LABELS[code] || code.toUpperCase(),
|
||||
confidence,
|
||||
status: confidence >= 75 ? 'high' : confidence >= 60 ? 'medium' : 'low'
|
||||
} : { code: 'unknown', label: 'Unknown', confidence: 0, status: 'unknown' },
|
||||
languageScores: scores,
|
||||
languageEvidence: [...algo.languageEvidence, ...laya.languageEvidence]
|
||||
};
|
||||
}
|
||||
|
||||
function applyLanguageMode(algo, result, mode) {
|
||||
if (mode === 'algo') return algo;
|
||||
if (mode === 'hybrid') return mergeHybrid(algo, result);
|
||||
return result ? layaDetection(result) : {
|
||||
language: { code: 'unknown', label: 'Unknown', confidence: 0, status: 'unknown' },
|
||||
languageScores: {},
|
||||
languageEvidence: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildLayaDescriptor(variant, variantKey, algo) {
|
||||
const fields = [
|
||||
['sourceId', variant.sourceId],
|
||||
['name', variant.name],
|
||||
['groupTitle', variant.groupTitle],
|
||||
['sourceName', variant.sourceName],
|
||||
['tvgId', variant.tvgId],
|
||||
['algorithmScores', JSON.stringify(algo.languageScores || {})]
|
||||
];
|
||||
return {
|
||||
id: variantKey,
|
||||
text: fields.filter(([, value]) => value !== undefined && value !== null && value !== '')
|
||||
.map(([key, value]) => `${key}: ${value}`).join('\n')
|
||||
};
|
||||
}
|
||||
|
||||
return { applyLanguageMode, buildLayaDescriptor, layaDetection, mergeHybrid, normalizeConfidence };
|
||||
});
|
||||
@ -2,10 +2,13 @@
|
||||
const detector = typeof module === 'object' && module.exports
|
||||
? require('./channelLanguageDetection.js')
|
||||
: root.ChannelLanguageDetection;
|
||||
const service = factory(detector);
|
||||
const fusion = typeof module === 'object' && module.exports
|
||||
? require('./channelLanguageFusion.js')
|
||||
: root.ChannelLanguageFusion;
|
||||
const service = factory(detector, fusion);
|
||||
if (typeof module === 'object' && module.exports) module.exports = service;
|
||||
if (root) root.ChannelVariantGrouping = service;
|
||||
})(typeof globalThis !== 'undefined' ? globalThis : this, function (languageDetection) {
|
||||
})(typeof globalThis !== 'undefined' ? globalThis : this, function (languageDetection, languageFusion) {
|
||||
const QUALITY_TOKENS = new Set([
|
||||
'4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264',
|
||||
'1080p', '1080i', '720p', '576p', '480p'
|
||||
@ -96,10 +99,14 @@
|
||||
});
|
||||
|
||||
return Array.from(buckets, ([fingerprint, rawVariants]) => {
|
||||
const variants = rawVariants.map(variant => ({
|
||||
...variant,
|
||||
...languageDetection.detectLanguage(variant, { sourceName: variant.sourceName })
|
||||
}));
|
||||
const variants = rawVariants.map(variant => {
|
||||
const algo = languageDetection.detectLanguage(variant, { sourceName: variant.sourceName });
|
||||
const result = options.languageResults?.get(getVariantKey(variant));
|
||||
return {
|
||||
...variant,
|
||||
...languageFusion.applyLanguageMode(algo, result, options.languageMode || 'algo')
|
||||
};
|
||||
});
|
||||
const visibleVariants = variants.filter(variant => !isHidden(variant));
|
||||
const selectableVariants = showHidden ? variants : visibleVariants;
|
||||
const languageCounts = selectableVariants.reduce((counts, variant) => {
|
||||
|
||||
@ -63,6 +63,7 @@ function getDefaultSettings() {
|
||||
lastVolume: 80,
|
||||
autoPlayNextEpisode: false,
|
||||
liveChannelListMode: 'grouped',
|
||||
languageDetectionMode: 'algo',
|
||||
forceProxy: false,
|
||||
forceTranscode: false, // Force Audio Transcode
|
||||
forceVideoTranscode: false, // Force Video Transcode
|
||||
|
||||
@ -135,6 +135,18 @@ function initSchema() {
|
||||
CREATE INDEX IF NOT EXISTS idx_history_user_item ON watch_history(user_id, item_id);
|
||||
`);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS laya_language_cache (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
mode TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
response TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_laya_cache_mode_model
|
||||
ON laya_language_cache(mode, model);
|
||||
`);
|
||||
|
||||
// Migration: Add source_id column if missing (for existing databases)
|
||||
try {
|
||||
db.exec(`ALTER TABLE watch_history ADD COLUMN source_id INTEGER`);
|
||||
@ -209,8 +221,32 @@ const favorites = {
|
||||
}
|
||||
};
|
||||
|
||||
const layaLanguageCache = {
|
||||
get(cacheKey) {
|
||||
const row = getDb().prepare(
|
||||
'SELECT response FROM laya_language_cache WHERE cache_key = ?'
|
||||
).get(cacheKey);
|
||||
return row ? JSON.parse(row.response) : null;
|
||||
},
|
||||
|
||||
set(cacheKey, mode, model, response) {
|
||||
getDb().prepare(`
|
||||
INSERT INTO laya_language_cache (cache_key, mode, model, response, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(cache_key) DO UPDATE SET
|
||||
response = excluded.response,
|
||||
created_at = excluded.created_at
|
||||
`).run(cacheKey, mode, model, JSON.stringify(response), Date.now());
|
||||
},
|
||||
|
||||
clear() {
|
||||
getDb().prepare('DELETE FROM laya_language_cache').run();
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getDb,
|
||||
initSchema,
|
||||
favorites
|
||||
favorites,
|
||||
layaLanguageCache
|
||||
};
|
||||
|
||||
@ -180,6 +180,7 @@ app.use('/api/probe', require('./routes/probe'));
|
||||
app.use('/api/subtitle', require('./routes/subtitle'));
|
||||
app.use('/api/settings', require('./routes/settings'));
|
||||
app.use('/api/history', require('./routes/history'));
|
||||
app.use('/api/laya', require('./routes/laya'));
|
||||
|
||||
// Version endpoint
|
||||
app.get('/api/version', (req, res) => {
|
||||
|
||||
42
server/routes/laya.js
Normal file
42
server/routes/laya.js
Normal file
@ -0,0 +1,42 @@
|
||||
const express = require('express');
|
||||
const { requireAuth, requireAdmin } = require('../auth');
|
||||
const { LayaRequestError, LayaService } = require('../services/layaService');
|
||||
|
||||
const router = express.Router();
|
||||
const laya = new LayaService();
|
||||
|
||||
function sendError(res, error) {
|
||||
const status = error instanceof LayaRequestError ? error.statusCode : 500;
|
||||
const message = error instanceof LayaRequestError ? error.message : 'Internal server error';
|
||||
res.status(status).json({ runtimeAvailable: false, error: message });
|
||||
}
|
||||
|
||||
router.get('/status', requireAuth, async (req, res) => {
|
||||
res.json(await laya.status());
|
||||
});
|
||||
|
||||
router.post('/model/download', requireAuth, requireAdmin, async (req, res) => {
|
||||
try {
|
||||
res.json(await laya.downloadModel(req.body?.model));
|
||||
} catch (error) {
|
||||
sendError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/model', requireAuth, requireAdmin, async (req, res) => {
|
||||
try {
|
||||
res.json(await laya.deleteModel());
|
||||
} catch (error) {
|
||||
sendError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/language/batch', requireAuth, async (req, res) => {
|
||||
try {
|
||||
res.json(await laya.detectBatch(req.body));
|
||||
} catch (error) {
|
||||
sendError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
195
server/services/layaService.js
Normal file
195
server/services/layaService.js
Normal file
@ -0,0 +1,195 @@
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
|
||||
const MODEL_DIR = path.resolve(__dirname, '..', '..', 'data', 'models', 'laya');
|
||||
const MAX_BATCH_ITEMS = 64;
|
||||
const MAX_BATCH_BYTES = 64 * 1024;
|
||||
const MAX_TEXT_LENGTH = 4096;
|
||||
const ALLOWED_MODELS = new Set(['english', 'multilingual', 'typed-decisions']);
|
||||
|
||||
class LayaRequestError extends Error {
|
||||
constructor(message, statusCode = 503) {
|
||||
super(message);
|
||||
this.name = 'LayaRequestError';
|
||||
this.statusCode = statusCode;
|
||||
}
|
||||
}
|
||||
|
||||
function validateBatchPayload(payload) {
|
||||
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
throw new LayaRequestError('Request body must be an object', 400);
|
||||
}
|
||||
if (!Array.isArray(payload.items) || payload.items.length === 0) {
|
||||
throw new LayaRequestError('items must be a non-empty array', 400);
|
||||
}
|
||||
if (Object.keys(payload).some(key => !['items', 'model'].includes(key))) {
|
||||
throw new LayaRequestError('Request body contains unsupported fields', 400);
|
||||
}
|
||||
if (payload.items.length > MAX_BATCH_ITEMS || Buffer.byteLength(JSON.stringify(payload)) > MAX_BATCH_BYTES) {
|
||||
throw new LayaRequestError('Batch payload is too large', 413);
|
||||
}
|
||||
payload.items.forEach((item, index) => {
|
||||
if (!item || typeof item !== 'object' || typeof item.text !== 'string' || !item.text.trim()) {
|
||||
throw new LayaRequestError(`items[${index}].text must be a non-empty string`, 400);
|
||||
}
|
||||
if (item.text.length > MAX_TEXT_LENGTH) {
|
||||
throw new LayaRequestError(`items[${index}].text is too long`, 413);
|
||||
}
|
||||
if (Object.keys(item).some(key => !['id', 'text'].includes(key))) {
|
||||
throw new LayaRequestError(`items[${index}] contains unsupported fields`, 400);
|
||||
}
|
||||
if (item.id !== undefined && !['string', 'number'].includes(typeof item.id)) {
|
||||
throw new LayaRequestError(`items[${index}].id must be a string or number`, 400);
|
||||
}
|
||||
});
|
||||
if (payload.model !== undefined && !ALLOWED_MODELS.has(payload.model)) {
|
||||
throw new LayaRequestError('Unsupported Laya model', 400);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function clearDirectoryContents(directory) {
|
||||
const root = path.resolve(directory);
|
||||
await fs.mkdir(root, { recursive: true });
|
||||
const entries = await fs.readdir(root);
|
||||
for (const entry of entries) {
|
||||
const target = path.resolve(root, entry);
|
||||
if (path.dirname(target) !== root) {
|
||||
throw new Error('Refusing to delete outside the Laya model directory');
|
||||
}
|
||||
await fs.rm(target, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
class LayaService {
|
||||
constructor(options = {}) {
|
||||
this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000';
|
||||
this.modelDir = path.resolve(options.modelDir || MODEL_DIR);
|
||||
this.fetch = options.fetch || globalThis.fetch;
|
||||
this.cache = options.cache === undefined
|
||||
? require('../db/sqlite').layaLanguageCache
|
||||
: options.cache;
|
||||
this.timeoutMs = options.timeoutMs || 3000;
|
||||
}
|
||||
|
||||
async request(endpoint, options = {}) {
|
||||
if (typeof this.fetch !== 'function') throw new LayaRequestError('Laya runtime client is unavailable');
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
const headers = { 'content-type': 'application/json' };
|
||||
if (process.env.LAYA_API_KEY) headers.authorization = `Bearer ${process.env.LAYA_API_KEY}`;
|
||||
try {
|
||||
const response = await this.fetch(`${this.baseUrl}${endpoint}`, {
|
||||
...options,
|
||||
headers: { ...headers, ...options.headers },
|
||||
signal: controller.signal
|
||||
});
|
||||
const data = await response.json().catch(() => null);
|
||||
if (!response.ok) throw new LayaRequestError(`Laya runtime returned HTTP ${response.status}`, 502);
|
||||
return data;
|
||||
} catch (error) {
|
||||
if (error instanceof LayaRequestError) throw error;
|
||||
throw new LayaRequestError('Laya runtime is unavailable');
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async status() {
|
||||
await fs.mkdir(this.modelDir, { recursive: true });
|
||||
const modelPresent = (await fs.readdir(this.modelDir)).length > 0;
|
||||
try {
|
||||
const runtime = await this.request('/health', { method: 'GET' });
|
||||
return { runtimeAvailable: true, modelPresent, runtime };
|
||||
} catch (error) {
|
||||
return { runtimeAvailable: false, modelPresent, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async downloadModel(model = 'multilingual') {
|
||||
if (!ALLOWED_MODELS.has(model)) throw new LayaRequestError('Unsupported Laya model', 400);
|
||||
await fs.mkdir(this.modelDir, { recursive: true });
|
||||
await this.request('/v1/systemone', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
state: 'Bonjour',
|
||||
model,
|
||||
questions: {
|
||||
language: {
|
||||
type: 'choice',
|
||||
instructions: 'Identify the language of this text.',
|
||||
criteria: { fr: 'French', en: 'English', es: 'Spanish', other: 'Another language' }
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
return this.status();
|
||||
}
|
||||
|
||||
async deleteModel() {
|
||||
await clearDirectoryContents(this.modelDir);
|
||||
if (this.cache) this.cache.clear();
|
||||
return { deleted: true, modelPresent: false };
|
||||
}
|
||||
|
||||
cacheKey(text, mode, model) {
|
||||
return crypto.createHash('sha256').update(`${mode}\0${model}\0${text}`).digest('hex');
|
||||
}
|
||||
|
||||
async detectBatch(payload) {
|
||||
validateBatchPayload(payload);
|
||||
const model = payload.model || 'multilingual';
|
||||
const mode = 'laya';
|
||||
const results = new Array(payload.items.length);
|
||||
const misses = [];
|
||||
|
||||
payload.items.forEach((item, index) => {
|
||||
const key = this.cacheKey(item.text, mode, model);
|
||||
const cached = this.cache ? this.cache.get(key) : null;
|
||||
if (cached) results[index] = { ...cached, cached: true };
|
||||
else misses.push({ item, index, key });
|
||||
});
|
||||
|
||||
if (misses.length) {
|
||||
const response = await this.request('/v1/systemone/batch', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
states: misses.map(({ item }) => item.text),
|
||||
model,
|
||||
questions: {
|
||||
language: {
|
||||
type: 'choice',
|
||||
instructions: 'Identify the language of this text.',
|
||||
criteria: { fr: 'French', en: 'English', es: 'Spanish', other: 'Another language' }
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
if (!response || !Array.isArray(response.results) || response.results.length !== misses.length) {
|
||||
throw new LayaRequestError('Laya runtime returned an invalid batch response', 502);
|
||||
}
|
||||
misses.forEach(({ item, index, key }, responseIndex) => {
|
||||
const answer = response.results[responseIndex]?.answers?.language;
|
||||
const result = {
|
||||
id: item.id ?? index,
|
||||
language: answer?.choice || 'unknown',
|
||||
confidence: answer?.answer_confidence ?? answer?.confidence ?? 0,
|
||||
cached: false
|
||||
};
|
||||
results[index] = result;
|
||||
if (this.cache) this.cache.set(key, mode, model, { ...result, cached: undefined });
|
||||
});
|
||||
}
|
||||
return { runtimeAvailable: true, model, results };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ALLOWED_MODELS,
|
||||
LayaRequestError,
|
||||
LayaService,
|
||||
MODEL_DIR,
|
||||
clearDirectoryContents,
|
||||
validateBatchPayload
|
||||
};
|
||||
41
test/channelLanguageFusion.test.js
Normal file
41
test/channelLanguageFusion.test.js
Normal file
@ -0,0 +1,41 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fusion = require('../public/js/services/channelLanguageFusion.js');
|
||||
|
||||
const algo = {
|
||||
language: { code: 'fr', label: 'French', confidence: 70, status: 'medium' },
|
||||
languageScores: { fr: 70, en: 30 },
|
||||
languageEvidence: [{ code: 'fr', weight: 60, type: 'name-prefix', value: 'FR' }]
|
||||
};
|
||||
|
||||
test('Laya-only mode maps backend confidence and language', () => {
|
||||
const result = fusion.applyLanguageMode(algo, { language: 'en', confidence: 0.88 }, 'laya');
|
||||
assert.deepEqual(result.language, { code: 'en', label: 'English', confidence: 88, status: 'high' });
|
||||
assert.equal(result.languageEvidence[0].type, 'laya');
|
||||
});
|
||||
|
||||
test('hybrid mode combines Laya and algorithm evidence', () => {
|
||||
const result = fusion.applyLanguageMode(algo, { language: 'en', confidence: 0.9 }, 'hybrid');
|
||||
assert.equal(result.language.code, 'en');
|
||||
assert.ok(result.languageScores.en > result.languageScores.fr);
|
||||
assert.deepEqual(result.languageEvidence.map(item => item.type), ['name-prefix', 'laya']);
|
||||
});
|
||||
|
||||
test('hybrid mode safely preserves algorithm output without Laya', () => {
|
||||
assert.equal(fusion.applyLanguageMode(algo, null, 'hybrid'), algo);
|
||||
});
|
||||
|
||||
test('batch descriptors contain only safe metadata in the backend id/text contract', () => {
|
||||
const descriptor = fusion.buildLayaDescriptor({
|
||||
sourceId: 4,
|
||||
name: '|FR| Sports',
|
||||
groupTitle: 'Sports',
|
||||
sourceName: 'Provider',
|
||||
tvgId: 'sports.fr',
|
||||
url: 'https://user:secret@example.test/live'
|
||||
}, '4:channel-1', algo);
|
||||
assert.deepEqual(Object.keys(descriptor), ['id', 'text']);
|
||||
assert.equal(descriptor.id, '4:channel-1');
|
||||
assert.match(descriptor.text, /sourceId: 4/);
|
||||
assert.doesNotMatch(descriptor.text, /secret|https:\/\//);
|
||||
});
|
||||
@ -59,3 +59,19 @@ test('favorites group updates are disabled for flat lists', () => {
|
||||
assert.equal(grouping.shouldUseFavoritesGroup('flat'), false);
|
||||
assert.equal(grouping.shouldUseFavoritesGroup('grouped'), true);
|
||||
});
|
||||
|
||||
test('Laya language results participate in variant selection', () => {
|
||||
const lowQualityFrench = channel('1', 'Sports One SD');
|
||||
const highQualityEnglish = channel('2', 'Sports One 4K');
|
||||
const languageResults = new Map([
|
||||
[grouping.getVariantKey(lowQualityFrench), { language: 'fr', confidence: 0.92 }],
|
||||
[grouping.getVariantKey(highQualityEnglish), { language: 'en', confidence: 0.94 }]
|
||||
]);
|
||||
const logical = grouping.groupChannels([lowQualityFrench, highQualityEnglish], {
|
||||
languageMode: 'laya',
|
||||
languageResults,
|
||||
preferredLanguage: 'fr'
|
||||
})[0];
|
||||
assert.equal(logical.preferredVariant.id, '1');
|
||||
assert.deepEqual(logical.availableLanguages.map(item => item.code), ['fr', 'en']);
|
||||
});
|
||||
|
||||
44
test/layaService.test.js
Normal file
44
test/layaService.test.js
Normal file
@ -0,0 +1,44 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('fs/promises');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const {
|
||||
LayaRequestError,
|
||||
LayaService,
|
||||
clearDirectoryContents,
|
||||
validateBatchPayload
|
||||
} = require('../server/services/layaService');
|
||||
|
||||
test('status reports an unavailable runtime without throwing', async () => {
|
||||
const modelDir = await fs.mkdtemp(path.join(os.tmpdir(), 'nodecast-laya-status-'));
|
||||
const service = new LayaService({
|
||||
modelDir,
|
||||
cache: null,
|
||||
fetch: async () => { throw new Error('connection refused'); }
|
||||
});
|
||||
const status = await service.status();
|
||||
assert.equal(status.runtimeAvailable, false);
|
||||
assert.equal(status.modelPresent, false);
|
||||
});
|
||||
|
||||
test('model deletion remains limited to the configured model directory', async () => {
|
||||
const parent = await fs.mkdtemp(path.join(os.tmpdir(), 'nodecast-laya-delete-'));
|
||||
const modelDir = path.join(parent, 'models');
|
||||
const sentinel = path.join(parent, 'keep.txt');
|
||||
await fs.mkdir(path.join(modelDir, 'nested'), { recursive: true });
|
||||
await fs.writeFile(path.join(modelDir, 'nested', 'model.bin'), 'model');
|
||||
await fs.writeFile(sentinel, 'keep');
|
||||
|
||||
await clearDirectoryContents(modelDir);
|
||||
|
||||
assert.deepEqual(await fs.readdir(modelDir), []);
|
||||
assert.equal(await fs.readFile(sentinel, 'utf8'), 'keep');
|
||||
});
|
||||
|
||||
test('batch validation rejects oversized item counts', () => {
|
||||
assert.throws(
|
||||
() => validateBatchPayload({ items: Array.from({ length: 65 }, (_, id) => ({ id, text: 'channel' })) }),
|
||||
error => error instanceof LayaRequestError && error.statusCode === 413
|
||||
);
|
||||
});
|
||||
@ -5,3 +5,7 @@ const { getDefaultSettings } = require('../server/db');
|
||||
test('defaults Live TV channel lists to grouped mode', () => {
|
||||
assert.equal(getDefaultSettings().liveChannelListMode, 'grouped');
|
||||
});
|
||||
|
||||
test('defaults language detection to the safe algorithmic mode', () => {
|
||||
assert.equal(getDefaultSettings().languageDetectionMode, 'algo');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user