a447d0dfe3
Run Local AI language detection during full rebuilds after identity classification and restore variant expansion with a visible playing stream indicator. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
703 lines
31 KiB
JavaScript
703 lines
31 KiB
JavaScript
const crypto = require('crypto');
|
|
const { pairKey, scoreCandidate, identity } = require('./liveGroupingScoring');
|
|
const { detectLanguage } = require('./channelLanguage');
|
|
const { matchesSearch } = require('./flexibleChannelSearch');
|
|
const iptvOrgCatalog = require('./iptvOrgCatalogService');
|
|
const epgMatching = require('./epgMatchingService');
|
|
|
|
const MAX_SYNCHRONOUS_ENRICHMENTS = 250;
|
|
const DEFAULT_PAGE_LIMIT = 100;
|
|
const MAX_PAGE_LIMIT = 200;
|
|
|
|
const QUALITY_TOKENS = new Set([
|
|
'4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264',
|
|
'1080p', '1080i', '720p', '576p', '480p'
|
|
]);
|
|
const REGION_PREFIX = /^(?:(?:\|\s*[a-z]{2,3}\s*\|)|(?:[a-z]{2,3}(?:-[a-z]{2,3})?\s*\|))\s*/i;
|
|
const BEIN_SUFFIX_TOKENS = new Set(['fr', 'en', 'thai', 'tod', 'appel', 'app']);
|
|
const BEIN_TIMESHIFT_TOKENS = new Set(['2h', '4h', '8h', '12h']);
|
|
|
|
function words(value) {
|
|
return String(value || '')
|
|
.normalize('NFKD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toLowerCase()
|
|
.replace(REGION_PREFIX, '')
|
|
.replace(/[|()[\]{}_.,:;+/\\-]+/g, ' ')
|
|
.split(/\s+/)
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function normalizeChannelName(name) {
|
|
const tokens = words(name).filter(token => !QUALITY_TOKENS.has(token));
|
|
const brandIndex = tokens.findIndex((token, index) => token === 'bein' && tokens[index + 1] === 'sports');
|
|
if (brandIndex >= 0) {
|
|
const branded = tokens.slice(brandIndex);
|
|
const numberedBaseLength = branded[2] === 'max' && /^\d+$/.test(branded[3])
|
|
? 4
|
|
: (/^\d+$/.test(branded[2]) ? 3 : 0);
|
|
if (numberedBaseLength) {
|
|
const suffix = branded.slice(numberedBaseLength);
|
|
const removable = suffix.every(token =>
|
|
BEIN_SUFFIX_TOKENS.has(token) || BEIN_TIMESHIFT_TOKENS.has(token)
|
|
);
|
|
if (removable) return branded.slice(0, numberedBaseLength).join(' ');
|
|
}
|
|
return branded.join(' ').trim();
|
|
}
|
|
return tokens.join(' ').trim();
|
|
}
|
|
|
|
function displayNameForFingerprint(fingerprint) {
|
|
return fingerprint.split(' ').map(token => {
|
|
if (token === 'bein') return 'beIN';
|
|
if (/^\d+$/.test(token)) return token;
|
|
return token.charAt(0).toUpperCase() + token.slice(1);
|
|
}).join(' ');
|
|
}
|
|
|
|
function qualityForName(name) {
|
|
const tokens = words(name);
|
|
if (tokens.includes('4k') || tokens.includes('uhd')) return { label: '4K', rank: 4 };
|
|
if (tokens.includes('fhd') || tokens.includes('fullhd') || tokens.includes('1080p') || tokens.includes('1080i')) {
|
|
return { label: 'Full HD', rank: 3 };
|
|
}
|
|
if (tokens.includes('hd') || tokens.includes('720p')) return { label: 'HD', rank: 2 };
|
|
if (tokens.includes('sd') || tokens.includes('576p') || tokens.includes('480p')) return { label: 'SD', rank: 0 };
|
|
return { label: 'Standard', rank: 1 };
|
|
}
|
|
|
|
function stableHash(value) {
|
|
return crypto.createHash('sha256').update(value).digest('hex');
|
|
}
|
|
|
|
function languageInputHash(item) {
|
|
return stableHash(`${item.name || ''}\0${item.category_name || ''}\0${item.tvg_id || ''}`);
|
|
}
|
|
|
|
function hasPersistedLanguage(item) {
|
|
if (!/^[a-z]{2,3}$/i.test(item.language_code || '') || String(item.language_code).toLowerCase() === 'und') return false;
|
|
if (!item.language_detector || !item.language_input_hash) return false;
|
|
const persistedHash = String(item.language_input_hash);
|
|
return !/^[a-f0-9]{64}$/i.test(persistedHash) || persistedHash === languageInputHash(item);
|
|
}
|
|
|
|
function parseNullableData(value) {
|
|
if (value == null) return null;
|
|
return parseData(value);
|
|
}
|
|
|
|
function projectCatalog(items, options = {}) {
|
|
const candidateBuckets = new Map();
|
|
for (const item of items) {
|
|
const fallback = `channel-${item.source_id}:${item.item_id}`;
|
|
const fingerprint = normalizeChannelName(item.name) || fallback;
|
|
const quality = qualityForName(item.name);
|
|
const projected = {
|
|
...item,
|
|
fingerprint,
|
|
groupingInputHash: stableHash(`${item.name || ''}\0${item.category_id || ''}`),
|
|
qualityLabel: quality.label,
|
|
qualityRank: quality.rank
|
|
};
|
|
if (hasPersistedLanguage(item)) {
|
|
projected.languageCode = item.language_code;
|
|
projected.languageConfidence = item.language_confidence;
|
|
projected.languageScores = parseNullableData(item.language_scores);
|
|
projected.languageEvidence = parseNullableData(item.language_evidence);
|
|
projected.languageDetector = item.language_detector;
|
|
projected.languageInputHash = item.language_input_hash;
|
|
} else {
|
|
const language = detectLanguage({ ...projected, data: parseData(projected.data) });
|
|
projected.languageCode = language.code;
|
|
projected.languageConfidence = language.confidence;
|
|
projected.languageScores = language.scores;
|
|
projected.languageEvidence = language.evidence;
|
|
projected.languageDetector = 'algorithm';
|
|
projected.languageInputHash = languageInputHash(projected);
|
|
}
|
|
const itemIdentity = identity(fingerprint);
|
|
const candidateKey = itemIdentity.family && itemIdentity.number
|
|
? `${itemIdentity.family}:${itemIdentity.number}`
|
|
: fingerprint;
|
|
if (!candidateBuckets.has(candidateKey)) candidateBuckets.set(candidateKey, []);
|
|
const groups = candidateBuckets.get(candidateKey);
|
|
const matchingGroup = groups.find(group => group.some(member => scoreCandidate(member, projected, {
|
|
mode: options.mode || 'algo',
|
|
layaHint: options.identityHints?.get(pairKey(member, projected))
|
|
}).matched));
|
|
if (matchingGroup) matchingGroup.push(projected);
|
|
else groups.push([projected]);
|
|
}
|
|
|
|
return Array.from(candidateBuckets.values()).flat().map(variants => {
|
|
const fingerprint = variants.map(variant => variant.fingerprint).sort((a, b) => a.length - b.length || a.localeCompare(b))[0];
|
|
variants.sort((a, b) => b.qualityRank - a.qualityRank || a.source_id - b.source_id || String(a.item_id).localeCompare(String(b.item_id)));
|
|
variants.forEach((variant, index) => { variant.variantRank = index; });
|
|
return {
|
|
id: `logical:${stableHash(fingerprint).slice(0, 24)}`,
|
|
fingerprint,
|
|
displayName: displayNameForFingerprint(fingerprint),
|
|
preferredVariantId: variants[0].id,
|
|
variants
|
|
};
|
|
}).sort((a, b) => a.displayName.localeCompare(b.displayName));
|
|
}
|
|
|
|
function safeIdentityDescriptor(item) {
|
|
return {
|
|
name: String(item.name || '').slice(0, 300),
|
|
group: String(item.category_name || item.groupTitle || '').slice(0, 200),
|
|
source: String(item.source_name || '').slice(0, 120),
|
|
tvgId: String(item.tvg_id || '').slice(0, 160)
|
|
};
|
|
}
|
|
|
|
function identityCandidatePairs(items, options = {}) {
|
|
const hasScope = Object.hasOwn(options, 'scopeItemIds');
|
|
const scopeItemIds = hasScope
|
|
? new Set(Array.from(options.scopeItemIds || [], String))
|
|
: null;
|
|
const isScoped = item => !scopeItemIds
|
|
|| scopeItemIds.has(String(item.id))
|
|
|| scopeItemIds.has(String(item.item_id));
|
|
const buckets = new Map();
|
|
for (const item of items) {
|
|
const fingerprint = normalizeChannelName(item.name);
|
|
const itemIdentity = identity(fingerprint);
|
|
if (!itemIdentity.family || !itemIdentity.number) continue;
|
|
const key = `${itemIdentity.family}:${itemIdentity.number}`;
|
|
if (!buckets.has(key)) buckets.set(key, []);
|
|
buckets.get(key).push({ ...item, fingerprint });
|
|
}
|
|
const pairs = [];
|
|
for (const bucket of buckets.values()) {
|
|
for (let left = 0; left < bucket.length; left++) {
|
|
for (let right = left + 1; right < bucket.length; right++) {
|
|
if (!isScoped(bucket[left]) && !isScoped(bucket[right])) continue;
|
|
if (bucket[left].fingerprint === bucket[right].fingerprint) continue;
|
|
if (scoreCandidate(bucket[left], bucket[right], { mode: 'algo' }).blocker) continue;
|
|
pairs.push({
|
|
id: pairKey(bucket[left], bucket[right]),
|
|
left: safeIdentityDescriptor(bucket[left]),
|
|
right: safeIdentityDescriptor(bucket[right])
|
|
});
|
|
if (pairs.length >= 256) return pairs;
|
|
}
|
|
}
|
|
}
|
|
return pairs;
|
|
}
|
|
|
|
async function collectIdentityHints(items, mode, layaClient, options = {}) {
|
|
if (mode === 'algo' || !layaClient) return new Map();
|
|
const pairOptions = Object.hasOwn(options, 'scopeItemIds')
|
|
? { scopeItemIds: options.scopeItemIds }
|
|
: {};
|
|
const pairs = identityCandidatePairs(items, pairOptions);
|
|
const total = Math.ceil(pairs.length / 64);
|
|
let processed = 0;
|
|
const hints = new Map();
|
|
options.onProgress?.({ processed, total });
|
|
for (let index = 0; index < pairs.length; index += 64) {
|
|
const results = await layaClient.identityHintBatch(pairs.slice(index, index + 64));
|
|
results.forEach(result => hints.set(result.id, result));
|
|
processed += 1;
|
|
options.onProgress?.({ processed, total });
|
|
}
|
|
return hints;
|
|
}
|
|
|
|
async function collectOptionalIdentityHints(items, mode, layaClient, options = {}) {
|
|
if (mode === 'algo') return new Map();
|
|
try {
|
|
return await collectIdentityHints(items, mode, layaClient, options);
|
|
} catch (error) {
|
|
console.warn('[LiveProjection] Laya identity hints unavailable; continuing deterministically');
|
|
return new Map();
|
|
}
|
|
}
|
|
|
|
function analysisScopeItems(items, options = {}) {
|
|
if (options.analyzeAll === true) return items;
|
|
if (Object.hasOwn(options, 'scopeItemIds')) {
|
|
const ids = new Set(Array.from(options.scopeItemIds || [], String));
|
|
return items.filter(item => ids.has(String(item.id)) || ids.has(String(item.item_id)));
|
|
}
|
|
return items.filter(item => !hasPersistedLanguage(item));
|
|
}
|
|
|
|
function languageDescriptor(item) {
|
|
const parts = [item.name, item.category_name, item.tvg_id].filter(Boolean).map(String);
|
|
return { id: item.id, text: parts.join(' | ').slice(0, 4096) || String(item.id) };
|
|
}
|
|
|
|
async function collectOptionalLayaLanguages(items, layaClient, options = {}) {
|
|
if (!layaClient?.detectBatch || items.length === 0) return;
|
|
const total = Math.ceil(items.length / 64);
|
|
let processed = 0;
|
|
options.onProgress?.({ processed, total });
|
|
try {
|
|
for (let index = 0; index < items.length; index += 64) {
|
|
const batch = items.slice(index, index + 64);
|
|
const response = await layaClient.detectBatch({
|
|
items: batch.map(languageDescriptor),
|
|
model: 'multilingual'
|
|
});
|
|
const results = new Map((response?.results || []).map(result => [String(result.id), result]));
|
|
for (const item of batch) {
|
|
const result = results.get(String(item.id));
|
|
const code = String(result?.language || '').toLowerCase();
|
|
if (!/^[a-z]{2,3}$/.test(code) || code === 'und') continue;
|
|
const confidence = Number(result.confidence) || 0;
|
|
item.language_code = code;
|
|
item.language_confidence = confidence;
|
|
item.language_scores = JSON.stringify({ [code]: confidence });
|
|
item.language_evidence = JSON.stringify([
|
|
{ code, weight: confidence, type: 'laya', value: 'Local AI model' }
|
|
]);
|
|
item.language_detector = 'laya';
|
|
item.language_input_hash = languageInputHash(item);
|
|
}
|
|
processed += 1;
|
|
options.onProgress?.({ processed, total });
|
|
}
|
|
} catch (error) {
|
|
console.warn('[LiveProjection] Laya language detection unavailable; keeping persisted or algorithmic languages');
|
|
}
|
|
}
|
|
|
|
function catalogInputHash(items) {
|
|
return stableHash(items
|
|
.map(item => `${item.id}\0${item.name}\0${item.category_id || ''}`)
|
|
.sort()
|
|
.join('\n'));
|
|
}
|
|
|
|
function defaultDatabase() {
|
|
return require('../db/sqlite').getDb();
|
|
}
|
|
|
|
function projectionState(database = null) {
|
|
database ||= defaultDatabase();
|
|
return database.prepare('SELECT * FROM catalog_projection_state WHERE projection = ?').get('live') || {
|
|
projection: 'live', status: 'missing', started_at: null, completed_at: null,
|
|
input_hash: null, projected_count: 0, phase: 'idle', processed_count: 0, total_count: 0,
|
|
laya_processed_count: 0, laya_total_count: 0, error: null
|
|
};
|
|
}
|
|
|
|
function updateProjectionProgress(database, phase, progress = {}) {
|
|
database.prepare(`
|
|
UPDATE catalog_projection_state SET phase = ?,
|
|
laya_processed_count = COALESCE(?, laya_processed_count),
|
|
laya_total_count = COALESCE(?, laya_total_count)
|
|
WHERE projection = 'live'
|
|
`).run(phase, progress.layaProcessed ?? null, progress.layaTotal ?? null);
|
|
}
|
|
|
|
function markProjectionBuilding(database = null) {
|
|
database ||= defaultDatabase();
|
|
const total = database.prepare("SELECT COUNT(*) AS count FROM playlist_items WHERE type = 'live'").get().count;
|
|
const startedAt = Date.now();
|
|
database.prepare(`
|
|
INSERT INTO catalog_projection_state
|
|
(projection, status, started_at, completed_at, input_hash, projected_count, phase,
|
|
processed_count, total_count, laya_processed_count, laya_total_count, error)
|
|
VALUES ('live', 'building', ?, NULL, NULL, 0, 'preparing', 0, ?, 0, 0, NULL)
|
|
ON CONFLICT(projection) DO UPDATE SET status = 'building', started_at = excluded.started_at,
|
|
completed_at = NULL, phase = 'preparing', processed_count = 0,
|
|
total_count = excluded.total_count, laya_processed_count = 0,
|
|
laya_total_count = 0, error = NULL
|
|
`).run(startedAt, total);
|
|
return projectionState(database);
|
|
}
|
|
|
|
function loadProjectionItems(database) {
|
|
return database.prepare(`
|
|
SELECT p.id, p.source_id, p.item_id, p.name, p.category_id, p.data,
|
|
p.language_code, p.language_confidence, p.language_scores, p.language_evidence,
|
|
p.language_detector, p.language_input_hash, c.name AS category_name
|
|
FROM playlist_items p
|
|
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id
|
|
WHERE p.type = 'live'
|
|
ORDER BY p.id
|
|
`).all().map(item => {
|
|
const data = parseData(item.data);
|
|
return { ...item, tvg_id: data.tvgId || data.tvg_id || null };
|
|
});
|
|
}
|
|
|
|
function rebuild(database = null, options = {}) {
|
|
database ||= defaultDatabase();
|
|
const items = options.items || loadProjectionItems(database);
|
|
if (!options.stateAlreadyBuilding) markProjectionBuilding(database);
|
|
updateProjectionProgress(database, 'projecting');
|
|
const groups = projectCatalog(items, { mode: options.mode, identityHints: options.identityHints });
|
|
const inputHash = catalogInputHash(items);
|
|
updateProjectionProgress(database, 'writing');
|
|
|
|
try {
|
|
const apply = database.transaction(() => {
|
|
database.prepare('DELETE FROM logical_channels').run();
|
|
database.prepare(`
|
|
UPDATE playlist_items SET logical_channel_id = NULL, grouping_fingerprint = NULL,
|
|
grouping_input_hash = NULL, quality_label = NULL, quality_rank = NULL, variant_rank = NULL
|
|
WHERE type = 'live'
|
|
`).run();
|
|
const insertLogical = database.prepare(`
|
|
INSERT INTO logical_channels
|
|
(id, fingerprint, display_name, preferred_variant_id, variant_count, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
`);
|
|
const updateVariant = database.prepare(`
|
|
UPDATE playlist_items SET logical_channel_id = ?, grouping_fingerprint = ?,
|
|
grouping_input_hash = ?, quality_label = ?, quality_rank = ?, variant_rank = ?,
|
|
language_code = ?, language_confidence = ?, language_scores = ?,
|
|
language_evidence = ?, language_detector = ?, language_input_hash = ?
|
|
WHERE id = ? AND type = 'live'
|
|
`);
|
|
const now = Date.now();
|
|
for (const group of groups) {
|
|
insertLogical.run(group.id, group.fingerprint, group.displayName, group.preferredVariantId, group.variants.length, now, now);
|
|
for (const variant of group.variants) {
|
|
updateVariant.run(group.id, group.fingerprint, variant.groupingInputHash,
|
|
variant.qualityLabel, variant.qualityRank, variant.variantRank,
|
|
variant.languageCode, variant.languageConfidence,
|
|
variant.languageScores == null ? null : JSON.stringify(variant.languageScores),
|
|
variant.languageEvidence == null ? null : JSON.stringify(variant.languageEvidence),
|
|
variant.languageDetector, variant.languageInputHash, variant.id);
|
|
}
|
|
}
|
|
database.prepare(`
|
|
UPDATE catalog_projection_state SET status = 'success', phase = 'complete', completed_at = ?,
|
|
input_hash = ?, projected_count = ?, processed_count = ?, total_count = ?, error = NULL
|
|
WHERE projection = 'live'
|
|
`).run(Date.now(), inputHash, groups.length, items.length, items.length);
|
|
});
|
|
apply();
|
|
return projectionState(database);
|
|
} catch (error) {
|
|
const diagnosticCode = /^SQLITE_[A-Z_]+$/.test(error.code || '') ? error.code : 'INTERNAL';
|
|
console.error(`[LiveProjection] Rebuild transaction failed (${diagnosticCode}, ${error.name || 'Error'})`);
|
|
database.prepare(`
|
|
UPDATE catalog_projection_state SET status = 'error', phase = 'failed', completed_at = ?, error = ?
|
|
WHERE projection = 'live'
|
|
`).run(Date.now(), `Live catalog projection failed [${diagnosticCode}]`);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function fullRebuildOrchestrator(database = null, dependencies = {}) {
|
|
database ||= defaultDatabase();
|
|
const syncService = dependencies.syncService || require('./syncService');
|
|
const sourceRepository = dependencies.sources || require('../db').sources;
|
|
await syncService.refreshEnabledLiveSources({ rebuild: false });
|
|
await startRebuild(database, { analyzeAll: true }).promise;
|
|
await syncService.refreshEnabledEpgSources();
|
|
const epgSources = await sourceRepository.getByType('epg');
|
|
epgMatching.rebuildPersistedEpgMatches(database, epgSources);
|
|
return projectionState(database);
|
|
}
|
|
|
|
async function rebuildWithSignals(database = null, options = {}) {
|
|
database ||= defaultDatabase();
|
|
const items = loadProjectionItems(database);
|
|
const configured = options.mode || (await require('../db').settings.get()).languageDetectionMode || 'algo';
|
|
const mode = ['algo', 'hybrid', 'laya'].includes(configured) ? configured : 'algo';
|
|
let identityHints = new Map();
|
|
if (mode !== 'algo') {
|
|
const layaClient = options.layaClient || new (require('./layaService').LayaService)();
|
|
const scopedItems = analysisScopeItems(items, options);
|
|
const identityOptions = options.analyzeAll === true
|
|
? {}
|
|
: Object.hasOwn(options, 'scopeItemIds')
|
|
? { scopeItemIds: options.scopeItemIds }
|
|
: { scopeItemIds: scopedItems.map(item => item.id) };
|
|
identityHints = await collectOptionalIdentityHints(items, mode, layaClient, {
|
|
...identityOptions,
|
|
onProgress: ({ processed, total }) => updateProjectionProgress(database, 'laya_identity', {
|
|
layaProcessed: processed,
|
|
layaTotal: total
|
|
})
|
|
});
|
|
await collectOptionalLayaLanguages(scopedItems, layaClient, {
|
|
onProgress: ({ processed, total }) => updateProjectionProgress(database, 'laya_language', {
|
|
layaProcessed: processed,
|
|
layaTotal: total
|
|
})
|
|
});
|
|
}
|
|
return rebuild(database, { ...options, items, mode, identityHints });
|
|
}
|
|
|
|
let activeRebuild = null;
|
|
let activeState = null;
|
|
|
|
function startRebuild(database = null, options = {}) {
|
|
if (activeRebuild) return { alreadyRunning: true, state: activeState, promise: activeRebuild };
|
|
database ||= defaultDatabase();
|
|
const {
|
|
mark = markProjectionBuilding,
|
|
runner = rebuildWithSignals,
|
|
schedule = setImmediate,
|
|
...rebuildOptions
|
|
} = options;
|
|
activeState = mark(database);
|
|
activeRebuild = new Promise((resolve, reject) => {
|
|
schedule(() => {
|
|
Promise.resolve()
|
|
.then(() => runner(database, { ...rebuildOptions, stateAlreadyBuilding: true }))
|
|
.then(resolve, reject);
|
|
});
|
|
}).finally(() => {
|
|
activeRebuild = null;
|
|
activeState = null;
|
|
});
|
|
return { alreadyRunning: false, state: activeState, promise: activeRebuild };
|
|
}
|
|
|
|
const queueRebuild = (database = null) => startRebuild(database, { mode: 'algo' }).promise;
|
|
|
|
function parseData(value) {
|
|
try { return value ? JSON.parse(value) : {}; } catch { return {}; }
|
|
}
|
|
|
|
function pageBounds(page = 1, limit = DEFAULT_PAGE_LIMIT) {
|
|
const normalizedPage = Math.max(1, Number.parseInt(page, 10) || 1);
|
|
const normalizedLimit = Math.min(MAX_PAGE_LIMIT, Math.max(1, Number.parseInt(limit, 10) || DEFAULT_PAGE_LIMIT));
|
|
return { page: normalizedPage, limit: normalizedLimit, offset: (normalizedPage - 1) * normalizedLimit };
|
|
}
|
|
|
|
function pageMetadata(total, bounds) {
|
|
const nextOffset = bounds.offset + bounds.limit;
|
|
const hasMore = nextOffset < total;
|
|
return {
|
|
page: bounds.page,
|
|
limit: bounds.limit,
|
|
offset: bounds.offset,
|
|
total,
|
|
hasMore,
|
|
nextOffset: hasMore ? nextOffset : null
|
|
};
|
|
}
|
|
|
|
function listChannel(channel) {
|
|
const preferredVariant = channel.preferredVariant || channel.variants[0] || null;
|
|
const variants = [];
|
|
const addPreview = variant => {
|
|
if (variant && variants.length < 3 && !variants.some(candidate =>
|
|
candidate.id === variant.id && candidate.sourceId === variant.sourceId
|
|
)) variants.push(variant);
|
|
};
|
|
addPreview(preferredVariant);
|
|
channel.variants.filter(variant => variant.isFavorite).forEach(addPreview);
|
|
channel.variants.filter(variant => !variant.isHidden).forEach(addPreview);
|
|
return {
|
|
...channel,
|
|
preferredVariant,
|
|
variantCount: channel.variants.length,
|
|
variants
|
|
};
|
|
}
|
|
|
|
function getChannelPage({ sourceId = null, includeHidden = false, userId = null, q = '', languages = [], page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null, dependencies = {}) {
|
|
database ||= defaultDatabase();
|
|
const standardCatalog = dependencies.iptvOrgCatalog || iptvOrgCatalog;
|
|
const params = [];
|
|
let sourceFilter = '';
|
|
if (sourceId !== null) {
|
|
sourceFilter = 'AND p.source_id = ?';
|
|
params.push(sourceId);
|
|
}
|
|
const rows = database.prepare(`
|
|
SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
|
|
l.display_name, l.preferred_variant_id
|
|
FROM playlist_items p
|
|
JOIN logical_channels l ON l.id = p.logical_channel_id
|
|
LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
|
|
AND f.item_id = p.item_id AND f.item_type = 'channel'
|
|
WHERE p.type = 'live' ${sourceFilter}
|
|
ORDER BY l.display_name, p.variant_rank
|
|
`).all(userId ?? -1, ...params);
|
|
const categoryRows = sourceId === null
|
|
? database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live'").all()
|
|
: database.prepare("SELECT source_id, category_id, name, is_hidden FROM categories WHERE type = 'live' AND source_id = ?").all(sourceId);
|
|
const categories = new Map(categoryRows.map(category => [`${category.source_id}\0${category.category_id}`, category]));
|
|
for (const row of rows) {
|
|
const category = categories.get(`${row.source_id}\0${row.category_id}`);
|
|
row.category_name = category?.name || null;
|
|
row.category_hidden = category?.is_hidden || 0;
|
|
}
|
|
const serialized = serializeChannels(rows, includeHidden);
|
|
const availableLanguages = [...new Map(serialized.map(channel => [channel.language.code, channel.language])).values()]
|
|
.map(language => ({ code: language.code, short: language.short, label: language.label }))
|
|
.sort((left, right) => left.label.localeCompare(right.label));
|
|
const filtered = serialized.filter(channel =>
|
|
(!languages.length || languages.includes(channel.language.code)) && matchesSearch(channel, q)
|
|
);
|
|
const persistedEpg = new Map(database.prepare(`
|
|
SELECT logical_channel_id, language_code, source_id, channel_id, match_confidence, data
|
|
FROM live_epg_matches
|
|
`).all().map(match => [`${match.logical_channel_id}\0${match.language_code}`, match]));
|
|
for (const channel of filtered) {
|
|
const match = persistedEpg.get(`${channel.logicalChannelId}\0${channel.language.code}`);
|
|
if (!match) continue;
|
|
let data = {};
|
|
try { data = match.data ? JSON.parse(match.data) : {}; } catch {}
|
|
channel.epg = {
|
|
...channel.epg,
|
|
...data,
|
|
sourceId: match.source_id,
|
|
channelId: match.channel_id,
|
|
matchConfidence: match.match_confidence,
|
|
language: channel.language.code
|
|
};
|
|
}
|
|
const bounds = pageBounds(page, limit);
|
|
const paged = filtered.slice(bounds.offset, bounds.offset + bounds.limit);
|
|
// Catalog loading must stay bounded: both enrichers perform synchronous SQLite work,
|
|
// and EPG matching is especially expensive against large XMLTV catalogs. Enrich only
|
|
// a genuinely targeted, bounded search; base projection data remains usable otherwise.
|
|
const shouldEnrich = Boolean(String(q).trim()) && filtered.length <= MAX_SYNCHRONOUS_ENRICHMENTS;
|
|
const enriched = !shouldEnrich ? paged : paged.map(channel => {
|
|
const match = standardCatalog.findMatch(database, channel);
|
|
if (match) {
|
|
channel.standardName = match.name;
|
|
channel.standard = { provider: 'iptv-org', channelId: match.channelId, country: match.country, confidence: match.confidence };
|
|
}
|
|
return channel;
|
|
});
|
|
return {
|
|
channels: enriched.map(listChannel),
|
|
availableLanguages,
|
|
page: pageMetadata(filtered.length, bounds)
|
|
};
|
|
}
|
|
|
|
function getChannelVariants({ logicalChannelId, includeHidden = false, userId = null, page = 1, limit = DEFAULT_PAGE_LIMIT }, database = null) {
|
|
database ||= defaultDatabase();
|
|
const rows = database.prepare(`
|
|
SELECT p.*, CASE WHEN f.id IS NULL THEN 0 ELSE 1 END AS user_favorite,
|
|
l.display_name, l.preferred_variant_id, c.name AS category_name,
|
|
COALESCE(c.is_hidden, 0) AS category_hidden
|
|
FROM playlist_items p
|
|
JOIN logical_channels l ON l.id = p.logical_channel_id
|
|
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id
|
|
LEFT JOIN favorites f ON f.user_id = ? AND f.source_id = p.source_id
|
|
AND f.item_id = p.item_id AND f.item_type = 'channel'
|
|
WHERE p.type = 'live' AND p.logical_channel_id = ?
|
|
ORDER BY p.variant_rank, p.id
|
|
`).all(userId ?? -1, logicalChannelId);
|
|
const variants = serializeChannels(rows, includeHidden).flatMap(channel => channel.variants);
|
|
const bounds = pageBounds(page, limit);
|
|
return {
|
|
variants: variants.slice(bounds.offset, bounds.offset + bounds.limit),
|
|
page: pageMetadata(variants.length, bounds)
|
|
};
|
|
}
|
|
|
|
function checkNewStreams() {
|
|
return require('./syncService').refreshEnabledLiveSources({ rebuild: true });
|
|
}
|
|
|
|
function getChannels(options, database = null) {
|
|
return getChannelPage(options, database).channels;
|
|
}
|
|
|
|
function serializeChannels(rows, includeHidden = false) {
|
|
const groups = new Map();
|
|
for (const row of rows) {
|
|
const hidden = Boolean(row.is_hidden || row.category_hidden);
|
|
if (hidden && !includeHidden) continue;
|
|
const raw = parseData(row.data);
|
|
const categoryName = raw.groupTitle || row.category_name || 'Uncategorized';
|
|
const variant = {
|
|
...raw,
|
|
id: row.item_id,
|
|
sourceId: row.source_id,
|
|
name: row.name,
|
|
categoryId: row.category_id,
|
|
categoryName,
|
|
groupTitle: categoryName,
|
|
streamIcon: row.stream_icon,
|
|
streamUrl: row.stream_url,
|
|
qualityLabel: row.quality_label,
|
|
qualityRank: row.quality_rank,
|
|
variantRank: row.variant_rank,
|
|
isHidden: hidden,
|
|
isFavorite: Boolean(row.user_favorite)
|
|
};
|
|
const languageCode = row.language_code || 'und';
|
|
const languageKey = `${row.logical_channel_id}:${languageCode}`;
|
|
if (!groups.has(languageKey)) {
|
|
groups.set(languageKey, {
|
|
id: languageKey,
|
|
logicalChannelId: row.logical_channel_id,
|
|
fingerprint: row.grouping_fingerprint,
|
|
displayName: row.display_name,
|
|
standardName: row.display_name,
|
|
standard: { provider: 'local', channelId: null, confidence: 0 },
|
|
language: {
|
|
code: languageCode,
|
|
short: languageCode === 'und' ? 'UND' : languageCode.toUpperCase(),
|
|
label: languageCode === 'und' ? 'Unknown' : new Intl.DisplayNames(['en'], { type: 'language' }).of(languageCode),
|
|
confidence: Number(row.language_confidence || 0)
|
|
},
|
|
name: row.display_name,
|
|
variants: [],
|
|
isFavorite: false,
|
|
isHidden: true
|
|
});
|
|
}
|
|
const group = groups.get(languageKey);
|
|
group.variants.push(variant);
|
|
group.isFavorite ||= variant.isFavorite;
|
|
group.isHidden &&= variant.isHidden;
|
|
if (!group.preferredVariant || Number(variant.variantRank) < Number(group.preferredVariant.variantRank)) group.preferredVariant = variant;
|
|
}
|
|
return Array.from(groups.values()).map(group => {
|
|
group.preferredVariant ||= group.variants[0];
|
|
group.categoryId = group.preferredVariant?.categoryId || null;
|
|
group.categoryName = group.preferredVariant?.categoryName || 'Uncategorized';
|
|
group.groupTitle = group.preferredVariant?.groupTitle || group.categoryName;
|
|
group.selectedVariantId = group.preferredVariant
|
|
? `${group.preferredVariant.sourceId}:${group.preferredVariant.id}`
|
|
: null;
|
|
const epgChannelId = group.preferredVariant?.tvgId || group.preferredVariant?.epg_channel_id || null;
|
|
group.epg = {
|
|
sourceId: null,
|
|
channelId: epgChannelId,
|
|
language: group.language.code,
|
|
matchConfidence: epgChannelId ? 100 : 0
|
|
};
|
|
return group;
|
|
});
|
|
}
|
|
|
|
module.exports = {
|
|
catalogInputHash,
|
|
collectIdentityHints,
|
|
collectOptionalIdentityHints,
|
|
displayNameForFingerprint,
|
|
checkNewStreams,
|
|
fullRebuildOrchestrator,
|
|
getChannelPage,
|
|
getChannelVariants,
|
|
getChannels,
|
|
identityCandidatePairs,
|
|
normalizeChannelName,
|
|
markProjectionBuilding,
|
|
MAX_SYNCHRONOUS_ENRICHMENTS,
|
|
matchesSearch,
|
|
projectCatalog,
|
|
projectionState,
|
|
qualityForName,
|
|
queueRebuild,
|
|
rebuild,
|
|
rebuildWithSignals,
|
|
safeIdentityDescriptor,
|
|
serializeChannels,
|
|
startRebuild
|
|
};
|