feat(live): add language-aware flexible channel search (#23)

Add backend language detection, flexible live channel search, iptv-org catalog caching, and protected built-in EPG source registration.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-06 11:42:06 +02:00
parent a6759b116d
commit 2a1c745a26
11 changed files with 314 additions and 19 deletions

View File

@ -1,6 +1,7 @@
const fs = require('fs/promises');
const path = require('path');
const { existsSync, mkdirSync } = require('fs');
const { ensureBuiltinSources } = require('./services/builtinEpgRegistry');
// Ensure data directory exists (sync is fine for startup)
const dataDir = path.join(__dirname, '..', 'data');
@ -17,25 +18,25 @@ async function loadDb() {
try {
const fileContent = await fs.readFile(dbPath, 'utf-8');
const data = JSON.parse(fileContent);
return {
return ensureBuiltinSources({
sources: data.sources || [],
hiddenItems: data.hiddenItems || [],
favorites: data.favorites || [],
settings: data.settings || getDefaultSettings(),
users: data.users || [],
nextId: data.nextId || 1
};
});
} catch (error) {
if (error.code === 'ENOENT') {
// File doesn't exist, return default
return {
return ensureBuiltinSources({
sources: [],
hiddenItems: [],
favorites: [],
settings: getDefaultSettings(),
users: [],
nextId: 1
};
});
}
throw error;
}
@ -168,6 +169,11 @@ const sources = {
const index = db.sources.findIndex(s => s.id === parseInt(id));
if (index === -1) return null;
const existing = db.sources[index];
if (existing.builtinKey) {
const allowed = Object.keys(updates).every(key => key === 'enabled');
if (!allowed) throw Object.assign(new Error('Built-in sources can only be enabled or disabled'), { code: 'BUILTIN_SOURCE_READONLY' });
}
db.sources[index] = {
...db.sources[index],
...updates,
@ -179,6 +185,8 @@ const sources = {
async delete(id) {
const db = await loadDb();
const existing = db.sources.find(s => s.id === parseInt(id));
if (existing?.builtinKey) throw Object.assign(new Error('Built-in sources cannot be removed'), { code: 'BUILTIN_SOURCE_PROTECTED' });
db.sources = db.sources.filter(s => s.id !== parseInt(id));
// Also delete related hidden items and favorites
db.hiddenItems = db.hiddenItems.filter(h => h.source_id !== parseInt(id));

View File

@ -36,6 +36,24 @@ const LIVE_PROJECTION_SCHEMA_SQL = `
);
CREATE INDEX IF NOT EXISTS idx_logical_channels_fingerprint ON logical_channels(fingerprint);
CREATE TABLE IF NOT EXISTS iptv_org_channels (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
normalized_name TEXT NOT NULL,
alt_names JSON NOT NULL DEFAULT '[]',
country TEXT,
languages JSON NOT NULL DEFAULT '[]',
data JSON,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_iptv_org_channels_name ON iptv_org_channels(normalized_name);
CREATE TABLE IF NOT EXISTS iptv_org_cache_state (
resource TEXT PRIMARY KEY,
refreshed_at INTEGER NOT NULL,
item_count INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS catalog_projection_state (
projection TEXT PRIMARY KEY,
status TEXT NOT NULL,

View File

@ -1,6 +1,8 @@
const express = require('express');
const { requireAuth, requireAdmin } = require('../auth');
const liveProjection = require('../services/liveProjectionService');
const iptvOrgCatalog = require('../services/iptvOrgCatalogService');
const { getDb } = require('../db/sqlite');
const router = express.Router();
@ -31,14 +33,25 @@ router.get('/channels', requireAuth, async (req, res) => {
return res.status(400).json({ error: 'includeHidden must be true or false' });
}
const includeHidden = req.query.includeHidden === 'true';
const q = String(req.query.q || '').slice(0, 200);
const languages = String(req.query.languages || '').split(',').map(value => value.trim().toLowerCase()).filter(Boolean);
if (languages.some(code => !/^(?:[a-z]{2,3}|und)$/.test(code))) {
return res.status(400).json({ error: 'languages must be comma-separated language codes' });
}
try {
let state = liveProjection.projectionState();
if (state.status !== 'success' && state.status !== 'building') {
state = await liveProjection.startRebuild().promise;
}
const channels = liveProjection.getChannels({ sourceId, includeHidden, userId: req.user.id });
return res.json({ projection: serializeState(state), channels });
const allChannels = liveProjection.getChannels({ sourceId, includeHidden, userId: req.user.id });
const availableLanguages = [...new Map(allChannels.map(channel => [channel.language.code, channel.language])).values()]
.map(language => ({ code: language.code, short: language.short, label: language.label }))
.sort((left, right) => left.label.localeCompare(right.label));
const channels = allChannels.filter(channel =>
(!languages.length || languages.includes(channel.language.code)) && liveProjection.matchesSearch(channel, q)
);
return res.json({ projection: serializeState(state), filters: { availableLanguages }, channels });
} catch (error) {
console.error('[LiveProjection] Failed to serve projected channels:', error.message);
const state = liveProjection.projectionState();
@ -74,5 +87,16 @@ router.post('/projection/rebuild', requireAuth, requireAdmin, (req, res) => {
}
});
router.post('/projection/standards/refresh', requireAuth, requireAdmin, async (req, res) => {
try {
const result = await iptvOrgCatalog.refresh(getDb());
await liveProjection.queueRebuild();
return res.json(result);
} catch (error) {
console.error('[LiveProjection] iptv-org refresh failed:', error.message);
return res.status(502).json({ error: 'Unable to refresh the iptv-org catalog' });
}
});
module.exports = router;
module.exports.serializeState = serializeState;

View File

@ -93,6 +93,9 @@ router.put('/:id', async (req, res) => {
}
const { name, url, username, password } = req.body;
if (existing.builtinKey) {
return res.status(409).json({ error: 'Built-in sources can only be enabled or disabled', code: 'BUILTIN_SOURCE_READONLY' });
}
const updated = await sources.update(req.params.id, {
name: name || existing.name,
url: url || existing.url,
@ -116,6 +119,9 @@ router.delete('/:id', async (req, res) => {
if (!existing) {
return res.status(404).json({ error: 'Source not found' });
}
if (existing.builtinKey) {
return res.status(409).json({ error: 'Built-in sources cannot be removed', code: 'BUILTIN_SOURCE_PROTECTED' });
}
// Cascade delete: Clean up SQLite data for this source
const db = getDb();

View File

@ -0,0 +1,20 @@
const BUILTIN_EPG_SOURCES = Object.freeze([
Object.freeze({ builtinKey: 'epgpw-fr', type: 'epg', name: 'EPG.PW France', url: 'https://epg.pw/xmltv/epg_FR.xml.gz', provider: 'EPG.PW', language: 'fr', country: 'FR', priority: 100 }),
Object.freeze({ builtinKey: 'epgpw-us', type: 'epg', name: 'EPG.PW United States', url: 'https://epg.pw/xmltv/epg_US.xml.gz', provider: 'EPG.PW', language: 'en', country: 'US', priority: 90 })
]);
function protectedFields(source) {
return source?.builtinKey ? { readonly: true, removable: false } : { readonly: false, removable: true };
}
function ensureBuiltinSources(database) {
database.sources ||= [];
for (const definition of BUILTIN_EPG_SOURCES) {
const existing = database.sources.find(source => source.builtinKey === definition.builtinKey);
if (existing) Object.assign(existing, definition, protectedFields(definition));
else database.sources.push({ id: database.nextId++, ...definition, ...protectedFields(definition), enabled: false, created_at: new Date().toISOString(), updated_at: new Date().toISOString() });
}
return database;
}
module.exports = { BUILTIN_EPG_SOURCES, ensureBuiltinSources, protectedFields };

View File

@ -0,0 +1,50 @@
const LANGUAGES = Object.freeze({
fr: { code: 'fr', short: 'FR', label: 'French' },
en: { code: 'en', short: 'EN', label: 'English' },
es: { code: 'es', short: 'ES', label: 'Spanish' },
de: { code: 'de', short: 'DE', label: 'German' },
it: { code: 'it', short: 'IT', label: 'Italian' },
pt: { code: 'pt', short: 'PT', label: 'Portuguese' }
});
const CODE_ALIASES = Object.freeze({
fr: 'fr', fra: 'fr', fre: 'fr', french: 'fr', francais: 'fr', france: 'fr',
en: 'en', eng: 'en', english: 'en', anglais: 'en', uk: 'en', us: 'en',
es: 'es', spa: 'es', spanish: 'es', espanol: 'es', spain: 'es',
de: 'de', deu: 'de', ger: 'de', german: 'de', deutsch: 'de',
it: 'it', ita: 'it', italian: 'it', italiano: 'it',
pt: 'pt', por: 'pt', portuguese: 'pt', portugues: 'pt'
});
function normalize(value) {
return String(value || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
}
function detectLanguage(item) {
const data = item.data && typeof item.data === 'object' ? item.data : {};
const values = [item.name, item.category_name, data.groupTitle, data.tvgId, data.tvgName].filter(Boolean);
const scores = {};
const evidence = [];
const prefix = String(item.name || '').trim().match(/^(?:\|\s*([a-z-]{2,8})\s*\||\[\s*([a-z-]{2,8})\s*\]|([a-z-]{2,8})\s*\|)/i);
if (prefix) add(prefix[1] || prefix[2] || prefix[3], 70, 'name-prefix');
values.forEach((value, index) => {
const tokens = normalize(value).split(/[^a-z]+/).filter(Boolean);
for (const token of tokens) add(token, index === 0 ? 45 : 25, index === 0 ? 'name-token' : 'metadata');
});
function add(token, weight, type) {
const code = CODE_ALIASES[normalize(token)];
if (!code) return;
scores[code] = (scores[code] || 0) + weight;
evidence.push({ code, weight, type, value: String(token) });
}
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
if (!ranked.length) return { code: 'und', short: 'UND', label: 'Unknown', confidence: 0, scores, evidence };
const total = ranked.reduce((sum, [, score]) => sum + score, 0);
const confidence = Math.round(ranked[0][1] / total * 100);
if (confidence < 55 || (ranked[1] && ranked[0][1] - ranked[1][1] < 15)) {
return { code: 'und', short: 'UND', label: 'Unknown', confidence: 0, scores, evidence };
}
return { ...LANGUAGES[ranked[0][0]], confidence, scores, evidence };
}
module.exports = { CODE_ALIASES, LANGUAGES, detectLanguage, normalize };

View File

@ -0,0 +1,19 @@
function normalizeSearch(value) {
return String(value || '')
.normalize('NFKD').replace(/[\u0300-\u036f]/g, '')
.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
.split(/\s+/).filter(Boolean)
.map(token => token.length > 3 && token.endsWith('s') ? token.slice(0, -1) : token);
}
function matchesSearch(channel, query) {
const wanted = normalizeSearch(query);
if (!wanted.length) return true;
const corpus = normalizeSearch([
channel.displayName, channel.standardName, channel.fingerprint,
...(channel.variants || []).flatMap(variant => [variant.name, variant.tvgId, variant.categoryName])
].filter(Boolean).join(' '));
return wanted.every(token => corpus.some(candidate => candidate.includes(token)));
}
module.exports = { matchesSearch, normalizeSearch };

View File

@ -0,0 +1,65 @@
const API_ROOT = 'https://iptv-org.github.io/api';
function normalize(value) {
return String(value || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '')
.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
}
function parseJson(value, fallback = []) {
try { return JSON.parse(value); } catch { return fallback; }
}
async function refresh(database, fetchImpl = globalThis.fetch) {
if (typeof fetchImpl !== 'function') throw new Error('A fetch implementation is required');
const [channelsResponse, feedsResponse] = await Promise.all([
fetchImpl(`${API_ROOT}/channels.json`), fetchImpl(`${API_ROOT}/feeds.json`)
]);
if (!channelsResponse.ok || !feedsResponse.ok) throw new Error('iptv-org API refresh failed');
const channels = await channelsResponse.json();
const feeds = await feedsResponse.json();
const feedLanguages = new Map();
for (const feed of feeds) {
const list = feedLanguages.get(feed.channel) || new Set();
for (const language of feed.languages || []) list.add(String(language).toLowerCase());
feedLanguages.set(feed.channel, list);
}
const write = database.transaction(() => {
database.prepare('DELETE FROM iptv_org_channels').run();
const insert = database.prepare(`
INSERT INTO iptv_org_channels
(id, name, normalized_name, alt_names, country, languages, data, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`);
const now = Date.now();
for (const channel of channels) {
insert.run(channel.id, channel.name, normalize(channel.name), JSON.stringify(channel.alt_names || []),
channel.country || null, JSON.stringify([...feedLanguages.get(channel.id) || []]), JSON.stringify(channel), now);
}
database.prepare(`
INSERT INTO iptv_org_cache_state (resource, refreshed_at, item_count) VALUES ('catalog', ?, ?)
ON CONFLICT(resource) DO UPDATE SET refreshed_at = excluded.refreshed_at, item_count = excluded.item_count
`).run(now, channels.length);
});
write();
return { refreshedAt: Date.now(), itemCount: channels.length };
}
function findMatch(database, channel) {
const names = new Set([channel.displayName, channel.fingerprint, ...(channel.variants || []).map(variant => variant.name)].map(normalize).filter(Boolean));
if (!names.size) return null;
const direct = database.prepare(`SELECT * FROM iptv_org_channels WHERE normalized_name IN (${[...names].map(() => '?').join(',')})`).all(...names);
const candidates = direct.length ? direct : database.prepare('SELECT * FROM iptv_org_channels').all().filter(row =>
parseJson(row.alt_names).some(name => names.has(normalize(name)))
);
const language = channel.language?.code;
const compatible = candidates.find(row => language === 'und' || parseJson(row.languages).includes(language)) || candidates[0];
if (!compatible) return null;
return {
channelId: compatible.id,
name: compatible.name,
country: compatible.country,
confidence: names.has(compatible.normalized_name) ? 100 : 90
};
}
module.exports = { API_ROOT, findMatch, normalize, refresh };

View File

@ -1,5 +1,8 @@
const crypto = require('crypto');
const { pairKey, scoreCandidate, identity } = require('./liveGroupingScoring');
const { detectLanguage } = require('./channelLanguage');
const { matchesSearch } = require('./flexibleChannelSearch');
const iptvOrgCatalog = require('./iptvOrgCatalogService');
const QUALITY_TOKENS = new Set([
'4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264',
@ -76,6 +79,11 @@ function projectCatalog(items, options = {}) {
qualityLabel: quality.label,
qualityRank: quality.rank
};
const language = detectLanguage({ ...projected, data: parseData(projected.data) });
projected.languageCode = language.code;
projected.languageConfidence = language.confidence;
projected.languageScores = language.scores;
projected.languageEvidence = language.evidence;
const itemIdentity = identity(fingerprint);
const candidateKey = itemIdentity.family && itemIdentity.number
? `${itemIdentity.family}:${itemIdentity.number}`
@ -241,7 +249,9 @@ function rebuild(database = null, options = {}) {
database.prepare('DELETE FROM logical_channels').run();
database.prepare(`
UPDATE playlist_items SET logical_channel_id = NULL, grouping_fingerprint = NULL,
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL,
language_code = NULL, language_confidence = NULL, language_scores = NULL,
language_evidence = NULL, language_detector = NULL, language_input_hash = NULL
WHERE type = 'live'
`).run();
const insertLogical = database.prepare(`
@ -251,7 +261,9 @@ function rebuild(database = null, options = {}) {
`);
const updateVariant = database.prepare(`
UPDATE playlist_items SET logical_channel_id = ?, grouping_fingerprint = ?,
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?,
language_code = ?, language_confidence = ?, language_scores = ?,
language_evidence = ?, language_detector = 'algorithm', language_input_hash = ?
WHERE id = ? AND type = 'live'
`);
const now = Date.now();
@ -259,7 +271,10 @@ function rebuild(database = null, options = {}) {
insertLogical.run(group.id, group.fingerprint, group.displayName, group.preferredVariantId, group.variants.length, now, now);
for (const variant of group.variants) {
updateVariant.run(group.id, group.fingerprint, variant.groupingInputHash,
variant.qualityLabel, variant.qualityRank, variant.variantRank, variant.id);
variant.qualityLabel, variant.qualityRank, variant.variantRank,
variant.languageCode, variant.languageConfidence,
JSON.stringify(variant.languageScores), JSON.stringify(variant.languageEvidence),
stableHash(`${variant.name || ''}\0${variant.category_name || ''}\0${variant.tvg_id || ''}`), variant.id);
}
}
database.prepare(`
@ -330,7 +345,7 @@ function parseData(value) {
try { return value ? JSON.parse(value) : {}; } catch { return {}; }
}
function getChannels({ sourceId = null, includeHidden = false, userId = null }, database = null) {
function getChannels({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [] }, database = null) {
database ||= defaultDatabase();
const params = [];
let sourceFilter = '';
@ -350,7 +365,16 @@ function getChannels({ sourceId = null, includeHidden = false, userId = null },
WHERE p.type = 'live' ${sourceFilter}
ORDER BY l.display_name, p.variant_rank
`).all(userId ?? -1, ...params);
return serializeChannels(rows, includeHidden);
return serializeChannels(rows, includeHidden).map(channel => {
const match = iptvOrgCatalog.findMatch(database, channel);
if (match) {
channel.standardName = match.name;
channel.standard = { provider: 'iptv-org', channelId: match.channelId, country: match.country, confidence: match.confidence };
}
return channel;
}).filter(channel =>
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
);
}
function serializeChannels(rows, includeHidden = false) {
@ -376,22 +400,33 @@ function serializeChannels(rows, includeHidden = false) {
isHidden: hidden,
isFavorite: Boolean(row.user_favorite)
};
if (!groups.has(row.logical_channel_id)) {
groups.set(row.logical_channel_id, {
id: row.logical_channel_id,
const languageCode = row.language_code || 'und';
const languageKey = `${row.logical_channel_id}:${languageCode}`;
if (!groups.has(languageKey)) {
groups.set(languageKey, {
id: languageKey,
logicalChannelId: row.logical_channel_id,
fingerprint: row.grouping_fingerprint,
displayName: row.display_name,
standardName: row.display_name,
standard: { provider: 'local', channelId: null, confidence: 0 },
language: {
code: languageCode,
short: languageCode === 'und' ? 'UND' : languageCode.toUpperCase(),
label: languageCode === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(languageCode),
confidence: Number(row.language_confidence || 0)
},
name: row.display_name,
variants: [],
isFavorite: false,
isHidden: true
});
}
const group = groups.get(row.logical_channel_id);
const group = groups.get(languageKey);
group.variants.push(variant);
group.isFavorite ||= variant.isFavorite;
group.isHidden &&= variant.isHidden;
if (row.id === row.preferred_variant_id) group.preferredVariant = variant;
if (!group.preferredVariant || Number(variant.variantRank) < Number(group.preferredVariant.variantRank)) group.preferredVariant = variant;
}
return Array.from(groups.values()).map(group => {
group.preferredVariant ||= group.variants[0];
@ -401,6 +436,13 @@ function serializeChannels(rows, includeHidden = false) {
group.selectedVariantId = group.preferredVariant
? `${group.preferredVariant.sourceId}:${group.preferredVariant.id}`
: null;
const epgChannelId = group.preferredVariant?.tvgId || group.preferredVariant?.epg_channel_id || null;
group.epg = {
sourceId: null,
channelId: epgChannelId,
language: group.language.code,
matchConfidence: epgChannelId ? 100 : 0
};
return group;
});
}
@ -414,6 +456,7 @@ module.exports = {
identityCandidatePairs,
normalizeChannelName,
markProjectionBuilding,
matchesSearch,
projectCatalog,
projectionState,
qualityForName,

View File

@ -125,10 +125,10 @@ test('functional live detection chain keeps categories and language selection co
assert.deepEqual([...new Set(rawRows.map(row => row.grouping_fingerprint))], ['sports one']);
const apiChannels = liveProjection.getChannels({ includeHidden: true, userId: 7 }, db);
const sportsOne = apiChannels.find(channel => channel.fingerprint === 'sports one');
assert.ok(sportsOne);
const sportsOneViews = apiChannels.filter(channel => channel.fingerprint === 'sports one');
assert.deepEqual(sportsOneViews.map(channel => channel.language.code).sort(), ['en', 'fr']);
assert.deepEqual(
sportsOne.variants
sportsOneViews.flatMap(channel => channel.variants)
.map(variant => [variant.id, variant.categoryName, variant.isHidden, variant.isFavorite])
.sort((left, right) => left[0].localeCompare(right[0])),
[

42
test/t23Backend.test.js Normal file
View File

@ -0,0 +1,42 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const Database = require('better-sqlite3');
const { detectLanguage } = require('../server/services/channelLanguage');
const { matchesSearch, normalizeSearch } = require('../server/services/flexibleChannelSearch');
const { ensureBuiltinSources } = require('../server/services/builtinEpgRegistry');
const iptvOrg = require('../server/services/iptvOrgCatalogService');
const { LIVE_PROJECTION_SCHEMA_SQL } = require('../server/db/liveProjectionSchema');
test('flexible search ignores accents, punctuation, case and simple plurals', () => {
const channel = { displayName: 'beIN Sports 1', variants: [{ name: 'BéIN SPORTS 1 HD' }] };
for (const query of ['Bein Sport 1', 'bein sports 1', 'beIN Sport 1']) assert.equal(matchesSearch(channel, query), true);
assert.deepEqual(normalizeSearch('Chaînes-Sports'), ['chaine', 'sport']);
});
test('backend language detection uses und and separates explicit languages', () => {
assert.equal(detectLanguage({ name: 'Generic Sports' }).code, 'und');
assert.equal(detectLanguage({ name: '|FR| Sports One' }).code, 'fr');
assert.equal(detectLanguage({ name: '[UK] Sports One' }).code, 'en');
});
test('built-in EPG registry is idempotent and protected while remaining disabled by default', () => {
const data = ensureBuiltinSources({ sources: [], nextId: 1 });
ensureBuiltinSources(data);
assert.equal(data.sources.length, 2);
assert.ok(data.sources.every(source => source.type === 'epg' && source.readonly && !source.removable && !source.enabled));
});
test('iptv-org adapter refreshes a persistent cache and matches aliases by language', async () => {
const db = new Database(':memory:');
db.exec(`CREATE TABLE playlist_items (id TEXT PRIMARY KEY, logical_channel_id TEXT, grouping_fingerprint TEXT);`);
db.exec(LIVE_PROJECTION_SCHEMA_SQL);
const payloads = {
'channels.json': [{ id: 'BeInSports1.fr', name: 'beIN Sports 1', alt_names: ['Bein Sport 1'], country: 'FR' }],
'feeds.json': [{ channel: 'BeInSports1.fr', languages: ['fr'] }]
};
await iptvOrg.refresh(db, async url => ({ ok: true, json: async () => payloads[url.split('/').pop()] }));
const match = iptvOrg.findMatch(db, { displayName: 'Bein Sport 1', language: { code: 'fr' }, variants: [] });
assert.equal(match.channelId, 'BeInSports1.fr');
assert.equal(match.name, 'beIN Sports 1');
assert.equal(db.prepare("SELECT item_count FROM iptv_org_cache_state WHERE resource = 'catalog'").get().item_count, 1);
});