feat(live): add language-aware flexible channel search (#23)
Add backend language detection, flexible live channel search, iptv-org catalog caching, and protected built-in EPG source registration. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
20
server/services/builtinEpgRegistry.js
Normal file
20
server/services/builtinEpgRegistry.js
Normal file
@ -0,0 +1,20 @@
|
||||
const BUILTIN_EPG_SOURCES = Object.freeze([
|
||||
Object.freeze({ builtinKey: 'epgpw-fr', type: 'epg', name: 'EPG.PW France', url: 'https://epg.pw/xmltv/epg_FR.xml.gz', provider: 'EPG.PW', language: 'fr', country: 'FR', priority: 100 }),
|
||||
Object.freeze({ builtinKey: 'epgpw-us', type: 'epg', name: 'EPG.PW United States', url: 'https://epg.pw/xmltv/epg_US.xml.gz', provider: 'EPG.PW', language: 'en', country: 'US', priority: 90 })
|
||||
]);
|
||||
|
||||
function protectedFields(source) {
|
||||
return source?.builtinKey ? { readonly: true, removable: false } : { readonly: false, removable: true };
|
||||
}
|
||||
|
||||
function ensureBuiltinSources(database) {
|
||||
database.sources ||= [];
|
||||
for (const definition of BUILTIN_EPG_SOURCES) {
|
||||
const existing = database.sources.find(source => source.builtinKey === definition.builtinKey);
|
||||
if (existing) Object.assign(existing, definition, protectedFields(definition));
|
||||
else database.sources.push({ id: database.nextId++, ...definition, ...protectedFields(definition), enabled: false, created_at: new Date().toISOString(), updated_at: new Date().toISOString() });
|
||||
}
|
||||
return database;
|
||||
}
|
||||
|
||||
module.exports = { BUILTIN_EPG_SOURCES, ensureBuiltinSources, protectedFields };
|
||||
50
server/services/channelLanguage.js
Normal file
50
server/services/channelLanguage.js
Normal file
@ -0,0 +1,50 @@
|
||||
const LANGUAGES = Object.freeze({
|
||||
fr: { code: 'fr', short: 'FR', label: 'French' },
|
||||
en: { code: 'en', short: 'EN', label: 'English' },
|
||||
es: { code: 'es', short: 'ES', label: 'Spanish' },
|
||||
de: { code: 'de', short: 'DE', label: 'German' },
|
||||
it: { code: 'it', short: 'IT', label: 'Italian' },
|
||||
pt: { code: 'pt', short: 'PT', label: 'Portuguese' }
|
||||
});
|
||||
|
||||
const CODE_ALIASES = Object.freeze({
|
||||
fr: 'fr', fra: 'fr', fre: 'fr', french: 'fr', francais: 'fr', france: 'fr',
|
||||
en: 'en', eng: 'en', english: 'en', anglais: 'en', uk: 'en', us: 'en',
|
||||
es: 'es', spa: 'es', spanish: 'es', espanol: 'es', spain: 'es',
|
||||
de: 'de', deu: 'de', ger: 'de', german: 'de', deutsch: 'de',
|
||||
it: 'it', ita: 'it', italian: 'it', italiano: 'it',
|
||||
pt: 'pt', por: 'pt', portuguese: 'pt', portugues: 'pt'
|
||||
});
|
||||
|
||||
function normalize(value) {
|
||||
return String(value || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
|
||||
}
|
||||
|
||||
function detectLanguage(item) {
|
||||
const data = item.data && typeof item.data === 'object' ? item.data : {};
|
||||
const values = [item.name, item.category_name, data.groupTitle, data.tvgId, data.tvgName].filter(Boolean);
|
||||
const scores = {};
|
||||
const evidence = [];
|
||||
const prefix = String(item.name || '').trim().match(/^(?:\|\s*([a-z-]{2,8})\s*\||\[\s*([a-z-]{2,8})\s*\]|([a-z-]{2,8})\s*\|)/i);
|
||||
if (prefix) add(prefix[1] || prefix[2] || prefix[3], 70, 'name-prefix');
|
||||
values.forEach((value, index) => {
|
||||
const tokens = normalize(value).split(/[^a-z]+/).filter(Boolean);
|
||||
for (const token of tokens) add(token, index === 0 ? 45 : 25, index === 0 ? 'name-token' : 'metadata');
|
||||
});
|
||||
function add(token, weight, type) {
|
||||
const code = CODE_ALIASES[normalize(token)];
|
||||
if (!code) return;
|
||||
scores[code] = (scores[code] || 0) + weight;
|
||||
evidence.push({ code, weight, type, value: String(token) });
|
||||
}
|
||||
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]);
|
||||
if (!ranked.length) return { code: 'und', short: 'UND', label: 'Unknown', confidence: 0, scores, evidence };
|
||||
const total = ranked.reduce((sum, [, score]) => sum + score, 0);
|
||||
const confidence = Math.round(ranked[0][1] / total * 100);
|
||||
if (confidence < 55 || (ranked[1] && ranked[0][1] - ranked[1][1] < 15)) {
|
||||
return { code: 'und', short: 'UND', label: 'Unknown', confidence: 0, scores, evidence };
|
||||
}
|
||||
return { ...LANGUAGES[ranked[0][0]], confidence, scores, evidence };
|
||||
}
|
||||
|
||||
module.exports = { CODE_ALIASES, LANGUAGES, detectLanguage, normalize };
|
||||
19
server/services/flexibleChannelSearch.js
Normal file
19
server/services/flexibleChannelSearch.js
Normal file
@ -0,0 +1,19 @@
|
||||
function normalizeSearch(value) {
|
||||
return String(value || '')
|
||||
.normalize('NFKD').replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
|
||||
.split(/\s+/).filter(Boolean)
|
||||
.map(token => token.length > 3 && token.endsWith('s') ? token.slice(0, -1) : token);
|
||||
}
|
||||
|
||||
function matchesSearch(channel, query) {
|
||||
const wanted = normalizeSearch(query);
|
||||
if (!wanted.length) return true;
|
||||
const corpus = normalizeSearch([
|
||||
channel.displayName, channel.standardName, channel.fingerprint,
|
||||
...(channel.variants || []).flatMap(variant => [variant.name, variant.tvgId, variant.categoryName])
|
||||
].filter(Boolean).join(' '));
|
||||
return wanted.every(token => corpus.some(candidate => candidate.includes(token)));
|
||||
}
|
||||
|
||||
module.exports = { matchesSearch, normalizeSearch };
|
||||
65
server/services/iptvOrgCatalogService.js
Normal file
65
server/services/iptvOrgCatalogService.js
Normal file
@ -0,0 +1,65 @@
|
||||
const API_ROOT = 'https://iptv-org.github.io/api';
|
||||
|
||||
function normalize(value) {
|
||||
return String(value || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function parseJson(value, fallback = []) {
|
||||
try { return JSON.parse(value); } catch { return fallback; }
|
||||
}
|
||||
|
||||
async function refresh(database, fetchImpl = globalThis.fetch) {
|
||||
if (typeof fetchImpl !== 'function') throw new Error('A fetch implementation is required');
|
||||
const [channelsResponse, feedsResponse] = await Promise.all([
|
||||
fetchImpl(`${API_ROOT}/channels.json`), fetchImpl(`${API_ROOT}/feeds.json`)
|
||||
]);
|
||||
if (!channelsResponse.ok || !feedsResponse.ok) throw new Error('iptv-org API refresh failed');
|
||||
const channels = await channelsResponse.json();
|
||||
const feeds = await feedsResponse.json();
|
||||
const feedLanguages = new Map();
|
||||
for (const feed of feeds) {
|
||||
const list = feedLanguages.get(feed.channel) || new Set();
|
||||
for (const language of feed.languages || []) list.add(String(language).toLowerCase());
|
||||
feedLanguages.set(feed.channel, list);
|
||||
}
|
||||
const write = database.transaction(() => {
|
||||
database.prepare('DELETE FROM iptv_org_channels').run();
|
||||
const insert = database.prepare(`
|
||||
INSERT INTO iptv_org_channels
|
||||
(id, name, normalized_name, alt_names, country, languages, data, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
const now = Date.now();
|
||||
for (const channel of channels) {
|
||||
insert.run(channel.id, channel.name, normalize(channel.name), JSON.stringify(channel.alt_names || []),
|
||||
channel.country || null, JSON.stringify([...feedLanguages.get(channel.id) || []]), JSON.stringify(channel), now);
|
||||
}
|
||||
database.prepare(`
|
||||
INSERT INTO iptv_org_cache_state (resource, refreshed_at, item_count) VALUES ('catalog', ?, ?)
|
||||
ON CONFLICT(resource) DO UPDATE SET refreshed_at = excluded.refreshed_at, item_count = excluded.item_count
|
||||
`).run(now, channels.length);
|
||||
});
|
||||
write();
|
||||
return { refreshedAt: Date.now(), itemCount: channels.length };
|
||||
}
|
||||
|
||||
function findMatch(database, channel) {
|
||||
const names = new Set([channel.displayName, channel.fingerprint, ...(channel.variants || []).map(variant => variant.name)].map(normalize).filter(Boolean));
|
||||
if (!names.size) return null;
|
||||
const direct = database.prepare(`SELECT * FROM iptv_org_channels WHERE normalized_name IN (${[...names].map(() => '?').join(',')})`).all(...names);
|
||||
const candidates = direct.length ? direct : database.prepare('SELECT * FROM iptv_org_channels').all().filter(row =>
|
||||
parseJson(row.alt_names).some(name => names.has(normalize(name)))
|
||||
);
|
||||
const language = channel.language?.code;
|
||||
const compatible = candidates.find(row => language === 'und' || parseJson(row.languages).includes(language)) || candidates[0];
|
||||
if (!compatible) return null;
|
||||
return {
|
||||
channelId: compatible.id,
|
||||
name: compatible.name,
|
||||
country: compatible.country,
|
||||
confidence: names.has(compatible.normalized_name) ? 100 : 90
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { API_ROOT, findMatch, normalize, refresh };
|
||||
@ -1,5 +1,8 @@
|
||||
const crypto = require('crypto');
|
||||
const { pairKey, scoreCandidate, identity } = require('./liveGroupingScoring');
|
||||
const { detectLanguage } = require('./channelLanguage');
|
||||
const { matchesSearch } = require('./flexibleChannelSearch');
|
||||
const iptvOrgCatalog = require('./iptvOrgCatalogService');
|
||||
|
||||
const QUALITY_TOKENS = new Set([
|
||||
'4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264',
|
||||
@ -76,6 +79,11 @@ function projectCatalog(items, options = {}) {
|
||||
qualityLabel: quality.label,
|
||||
qualityRank: quality.rank
|
||||
};
|
||||
const language = detectLanguage({ ...projected, data: parseData(projected.data) });
|
||||
projected.languageCode = language.code;
|
||||
projected.languageConfidence = language.confidence;
|
||||
projected.languageScores = language.scores;
|
||||
projected.languageEvidence = language.evidence;
|
||||
const itemIdentity = identity(fingerprint);
|
||||
const candidateKey = itemIdentity.family && itemIdentity.number
|
||||
? `${itemIdentity.family}:${itemIdentity.number}`
|
||||
@ -241,7 +249,9 @@ function rebuild(database = null, options = {}) {
|
||||
database.prepare('DELETE FROM logical_channels').run();
|
||||
database.prepare(`
|
||||
UPDATE playlist_items SET logical_channel_id = NULL, grouping_fingerprint = NULL,
|
||||
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL
|
||||
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL,
|
||||
language_code = NULL, language_confidence = NULL, language_scores = NULL,
|
||||
language_evidence = NULL, language_detector = NULL, language_input_hash = NULL
|
||||
WHERE type = 'live'
|
||||
`).run();
|
||||
const insertLogical = database.prepare(`
|
||||
@ -251,7 +261,9 @@ function rebuild(database = null, options = {}) {
|
||||
`);
|
||||
const updateVariant = database.prepare(`
|
||||
UPDATE playlist_items SET logical_channel_id = ?, grouping_fingerprint = ?,
|
||||
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?
|
||||
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?,
|
||||
language_code = ?, language_confidence = ?, language_scores = ?,
|
||||
language_evidence = ?, language_detector = 'algorithm', language_input_hash = ?
|
||||
WHERE id = ? AND type = 'live'
|
||||
`);
|
||||
const now = Date.now();
|
||||
@ -259,7 +271,10 @@ function rebuild(database = null, options = {}) {
|
||||
insertLogical.run(group.id, group.fingerprint, group.displayName, group.preferredVariantId, group.variants.length, now, now);
|
||||
for (const variant of group.variants) {
|
||||
updateVariant.run(group.id, group.fingerprint, variant.groupingInputHash,
|
||||
variant.qualityLabel, variant.qualityRank, variant.variantRank, variant.id);
|
||||
variant.qualityLabel, variant.qualityRank, variant.variantRank,
|
||||
variant.languageCode, variant.languageConfidence,
|
||||
JSON.stringify(variant.languageScores), JSON.stringify(variant.languageEvidence),
|
||||
stableHash(`${variant.name || ''}\0${variant.category_name || ''}\0${variant.tvg_id || ''}`), variant.id);
|
||||
}
|
||||
}
|
||||
database.prepare(`
|
||||
@ -330,7 +345,7 @@ function parseData(value) {
|
||||
try { return value ? JSON.parse(value) : {}; } catch { return {}; }
|
||||
}
|
||||
|
||||
function getChannels({ sourceId = null, includeHidden = false, userId = null }, database = null) {
|
||||
function getChannels({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [] }, database = null) {
|
||||
database ||= defaultDatabase();
|
||||
const params = [];
|
||||
let sourceFilter = '';
|
||||
@ -350,7 +365,16 @@ function getChannels({ sourceId = null, includeHidden = false, userId = null },
|
||||
WHERE p.type = 'live' ${sourceFilter}
|
||||
ORDER BY l.display_name, p.variant_rank
|
||||
`).all(userId ?? -1, ...params);
|
||||
return serializeChannels(rows, includeHidden);
|
||||
return serializeChannels(rows, includeHidden).map(channel => {
|
||||
const match = iptvOrgCatalog.findMatch(database, channel);
|
||||
if (match) {
|
||||
channel.standardName = match.name;
|
||||
channel.standard = { provider: 'iptv-org', channelId: match.channelId, country: match.country, confidence: match.confidence };
|
||||
}
|
||||
return channel;
|
||||
}).filter(channel =>
|
||||
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
|
||||
);
|
||||
}
|
||||
|
||||
function serializeChannels(rows, includeHidden = false) {
|
||||
@ -376,22 +400,33 @@ function serializeChannels(rows, includeHidden = false) {
|
||||
isHidden: hidden,
|
||||
isFavorite: Boolean(row.user_favorite)
|
||||
};
|
||||
if (!groups.has(row.logical_channel_id)) {
|
||||
groups.set(row.logical_channel_id, {
|
||||
id: row.logical_channel_id,
|
||||
const languageCode = row.language_code || 'und';
|
||||
const languageKey = `${row.logical_channel_id}:${languageCode}`;
|
||||
if (!groups.has(languageKey)) {
|
||||
groups.set(languageKey, {
|
||||
id: languageKey,
|
||||
logicalChannelId: row.logical_channel_id,
|
||||
fingerprint: row.grouping_fingerprint,
|
||||
displayName: row.display_name,
|
||||
standardName: row.display_name,
|
||||
standard: { provider: 'local', channelId: null, confidence: 0 },
|
||||
language: {
|
||||
code: languageCode,
|
||||
short: languageCode === 'und' ? 'UND' : languageCode.toUpperCase(),
|
||||
label: languageCode === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(languageCode),
|
||||
confidence: Number(row.language_confidence || 0)
|
||||
},
|
||||
name: row.display_name,
|
||||
variants: [],
|
||||
isFavorite: false,
|
||||
isHidden: true
|
||||
});
|
||||
}
|
||||
const group = groups.get(row.logical_channel_id);
|
||||
const group = groups.get(languageKey);
|
||||
group.variants.push(variant);
|
||||
group.isFavorite ||= variant.isFavorite;
|
||||
group.isHidden &&= variant.isHidden;
|
||||
if (row.id === row.preferred_variant_id) group.preferredVariant = variant;
|
||||
if (!group.preferredVariant || Number(variant.variantRank) < Number(group.preferredVariant.variantRank)) group.preferredVariant = variant;
|
||||
}
|
||||
return Array.from(groups.values()).map(group => {
|
||||
group.preferredVariant ||= group.variants[0];
|
||||
@ -401,6 +436,13 @@ function serializeChannels(rows, includeHidden = false) {
|
||||
group.selectedVariantId = group.preferredVariant
|
||||
? `${group.preferredVariant.sourceId}:${group.preferredVariant.id}`
|
||||
: null;
|
||||
const epgChannelId = group.preferredVariant?.tvgId || group.preferredVariant?.epg_channel_id || null;
|
||||
group.epg = {
|
||||
sourceId: null,
|
||||
channelId: epgChannelId,
|
||||
language: group.language.code,
|
||||
matchConfidence: epgChannelId ? 100 : 0
|
||||
};
|
||||
return group;
|
||||
});
|
||||
}
|
||||
@ -414,6 +456,7 @@ module.exports = {
|
||||
identityCandidatePairs,
|
||||
normalizeChannelName,
|
||||
markProjectionBuilding,
|
||||
matchesSearch,
|
||||
projectCatalog,
|
||||
projectionState,
|
||||
qualityForName,
|
||||
|
||||
Reference in New Issue
Block a user