diff --git a/server/services/layaService.js b/server/services/layaService.js index 8939af1..eedf673 100644 --- a/server/services/layaService.js +++ b/server/services/layaService.js @@ -103,6 +103,22 @@ function initialDownloadState() { }; } +function sanitizeIdentityText(value, maxLength) { + return String(value || '') + .replace(/https?:\/\/\S+/gi, '[redacted]') + .replace(/\b(?:password|passwd|token|username|user)\s*[=:]\s*\S+/gi, '[redacted]') + .slice(0, maxLength); +} + +function sanitizeIdentityDescriptor(value) { + return { + name: sanitizeIdentityText(value?.name, 300), + group: sanitizeIdentityText(value?.group, 200), + source: sanitizeIdentityText(value?.source, 120), + tvgId: sanitizeIdentityText(value?.tvgId, 160) + }; +} + class LayaService { constructor(options = {}) { this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000'; @@ -300,6 +316,41 @@ class LayaService { } return { runtimeAvailable: true, model, results }; } + + async identityHintBatch(pairs) { + if (!Array.isArray(pairs) || pairs.length === 0 || pairs.length > MAX_BATCH_ITEMS) { + throw new LayaRequestError(`Identity hint batch must contain 1 to ${MAX_BATCH_ITEMS} pairs`, 400); + } + const response = await this.request('/v1/systemone/batch', { + method: 'POST', + body: JSON.stringify({ + states: pairs.map(pair => ({ + left: sanitizeIdentityDescriptor(pair.left), + right: sanitizeIdentityDescriptor(pair.right) + })), + model: 'multilingual', + questions: { + sameIdentity: { + type: 'noul', + instructions: 'Do these two TV channel descriptors identify the same numbered channel?' + } + } + }) + }); + if (!response || !Array.isArray(response.results) || response.results.length !== pairs.length) { + throw new LayaRequestError('Laya runtime returned an invalid identity batch response', 502); + } + return pairs.map((pair, index) => { + const answer = response.results[index]?.answers?.sameIdentity || {}; + return { + id: pair.id, + sameIdentityProbability: Number(answer.noul) || 0, + confidence: Number(answer.answer_confidence ?? answer.confidence) || 0, + family: answer.family || null, + number: answer.number || null + }; + }); + } } module.exports = { @@ -310,5 +361,6 @@ module.exports = { clearDirectoryContents, directorySize, initialDownloadState, + sanitizeIdentityDescriptor, validateBatchPayload }; diff --git a/server/services/liveGroupingScoring.js b/server/services/liveGroupingScoring.js new file mode 100644 index 0000000..5473b3b --- /dev/null +++ b/server/services/liveGroupingScoring.js @@ -0,0 +1,105 @@ +const MODE_WEIGHTS = Object.freeze({ + algo: Object.freeze({ algorithm: 80, structure: 15, metadata: 5, laya: 0 }), + hybrid: Object.freeze({ algorithm: 70, structure: 15, metadata: 5, laya: 10 }), + laya: Object.freeze({ algorithm: 55, structure: 15, metadata: 5, laya: 25 }) +}); +const PROTECTED_TOKENS = new Set(['max', 'news', 'premium', 'extra', 'xtra', 'kids', 'movies']); +const STRONG_FAMILIES = ['bein sports', 'sky sports', 'canal plus', 'eurosport', 'espn', 'dazn']; +const LAYA_CONFIDENCE_FLOOR = 0.65; +const MERGE_THRESHOLD = 70; + +function tokens(value) { + return String(value || '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); +} + +function strongFamily(value) { + const text = tokens(value).join(' '); + return STRONG_FAMILIES.find(family => text.includes(family)) || null; +} + +function identityNumber(value, family = strongFamily(value)) { + const list = tokens(value); + let start = 0; + if (family) { + const familyTokens = family.split(' '); + for (let index = 0; index <= list.length - familyTokens.length; index++) { + if (familyTokens.every((token, offset) => list[index + offset] === token)) { + start = index + familyTokens.length; + break; + } + } + } + return list.slice(start).find(token => /^\d+$/.test(token)) || null; +} + +function identity(value) { + const list = tokens(value); + const family = strongFamily(value); + return { + family, + number: identityNumber(value, family), + protectedTokens: new Set(list.filter(token => PROTECTED_TOKENS.has(token))) + }; +} + +function sameSet(left, right) { + return left.size === right.size && [...left].every(value => right.has(value)); +} + +function hardBlock(left, right) { + const a = identity(left.fingerprint || left.name); + const b = identity(right.fingerprint || right.name); + if (!sameSet(a.protectedTokens, b.protectedTokens)) return 'protected_token_mismatch'; + if (a.number && b.number && a.number !== b.number) return 'number_mismatch'; + if (a.family && b.family && a.family !== b.family) return 'strong_family_mismatch'; + return null; +} + +function pairKey(left, right) { + return [String(left.id), String(right.id)].sort().join('\0'); +} + +function layaSignal(hint, leftIdentity, rightIdentity) { + if (!hint || Number(hint.confidence) < LAYA_CONFIDENCE_FLOOR) return 0; + if (hint.number && leftIdentity.number && hint.number !== leftIdentity.number) return 0; + if (hint.number && rightIdentity.number && hint.number !== rightIdentity.number) return 0; + if (hint.family && leftIdentity.family && hint.family !== leftIdentity.family) return 0; + if (hint.family && rightIdentity.family && hint.family !== rightIdentity.family) return 0; + return Number(hint.sameIdentityProbability) >= 0.5 + ? Math.min(1, Math.max(0, Number(hint.confidence))) + : 0; +} + +function scoreCandidate(left, right, options = {}) { + const mode = MODE_WEIGHTS[options.mode] ? options.mode : 'algo'; + const weights = MODE_WEIGHTS[mode]; + const blocker = hardBlock(left, right); + if (blocker) return { matched: false, blocker, total: 0, signals: { algorithm: 0, structure: 0, metadata: 0, laya: 0 } }; + + const leftIdentity = identity(left.fingerprint || left.name); + const rightIdentity = identity(right.fingerprint || right.name); + const exact = left.fingerprint === right.fingerprint; + const sameStructure = Boolean(leftIdentity.family && leftIdentity.family === rightIdentity.family && + leftIdentity.number && leftIdentity.number === rightIdentity.number); + const algorithm = exact ? 1 : (mode !== 'algo' && sameStructure ? 0.6 : 0); + const structure = sameStructure ? 1 : 0; + const metadata = Boolean( + (left.category_id && left.category_id === right.category_id) || + (left.tvg_id && left.tvg_id === right.tvg_id) + ) ? 1 : 0; + const laya = exact ? 0 : layaSignal(options.layaHint, leftIdentity, rightIdentity); + const total = algorithm * weights.algorithm + structure * weights.structure + + metadata * weights.metadata + laya * weights.laya; + return { matched: total >= MERGE_THRESHOLD, blocker: null, total, signals: { algorithm, structure, metadata, laya } }; +} + +module.exports = { + LAYA_CONFIDENCE_FLOOR, + MERGE_THRESHOLD, + MODE_WEIGHTS, + PROTECTED_TOKENS, + hardBlock, + identity, + pairKey, + scoreCandidate +}; diff --git a/server/services/liveProjectionService.js b/server/services/liveProjectionService.js index 37681c5..e209834 100644 --- a/server/services/liveProjectionService.js +++ b/server/services/liveProjectionService.js @@ -1,4 +1,5 @@ const crypto = require('crypto'); +const { pairKey, scoreCandidate, identity } = require('./liveGroupingScoring'); const QUALITY_TOKENS = new Set([ '4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264', @@ -62,8 +63,8 @@ function stableHash(value) { return crypto.createHash('sha256').update(value).digest('hex'); } -function projectCatalog(items) { - const buckets = new Map(); +function projectCatalog(items, options = {}) { + const candidateBuckets = new Map(); for (const item of items) { const fallback = `channel-${item.source_id}:${item.item_id}`; const fingerprint = normalizeChannelName(item.name) || fallback; @@ -75,11 +76,22 @@ function projectCatalog(items) { qualityLabel: quality.label, qualityRank: quality.rank }; - if (!buckets.has(fingerprint)) buckets.set(fingerprint, []); - buckets.get(fingerprint).push(projected); + const itemIdentity = identity(fingerprint); + const candidateKey = itemIdentity.family && itemIdentity.number + ? `${itemIdentity.family}:${itemIdentity.number}` + : fingerprint; + if (!candidateBuckets.has(candidateKey)) candidateBuckets.set(candidateKey, []); + const groups = candidateBuckets.get(candidateKey); + const matchingGroup = groups.find(group => scoreCandidate(group[0], projected, { + mode: options.mode || 'algo', + layaHint: options.identityHints?.get(pairKey(group[0], projected)) + }).matched); + if (matchingGroup) matchingGroup.push(projected); + else groups.push([projected]); } - return Array.from(buckets, ([fingerprint, variants]) => { + return Array.from(candidateBuckets.values()).flat().map(variants => { + const fingerprint = variants.map(variant => variant.fingerprint).sort((a, b) => a.length - b.length || a.localeCompare(b))[0]; variants.sort((a, b) => b.qualityRank - a.qualityRank || a.source_id - b.source_id || String(a.item_id).localeCompare(String(b.item_id))); variants.forEach((variant, index) => { variant.variantRank = index; }); return { @@ -92,6 +104,54 @@ function projectCatalog(items) { }).sort((a, b) => a.displayName.localeCompare(b.displayName)); } +function safeIdentityDescriptor(item) { + return { + name: String(item.name || '').slice(0, 300), + group: String(item.category_name || item.groupTitle || '').slice(0, 200), + source: String(item.source_name || '').slice(0, 120), + tvgId: String(item.tvg_id || '').slice(0, 160) + }; +} + +function identityCandidatePairs(items) { + const buckets = new Map(); + for (const item of items) { + const fingerprint = normalizeChannelName(item.name); + const itemIdentity = identity(fingerprint); + if (!itemIdentity.family || !itemIdentity.number) continue; + const key = `${itemIdentity.family}:${itemIdentity.number}`; + if (!buckets.has(key)) buckets.set(key, []); + buckets.get(key).push({ ...item, fingerprint }); + } + const pairs = []; + for (const bucket of buckets.values()) { + for (let left = 0; left < bucket.length; left++) { + for (let right = left + 1; right < bucket.length; right++) { + if (bucket[left].fingerprint === bucket[right].fingerprint) continue; + if (scoreCandidate(bucket[left], bucket[right], { mode: 'algo' }).blocker) continue; + pairs.push({ + id: pairKey(bucket[left], bucket[right]), + left: safeIdentityDescriptor(bucket[left]), + right: safeIdentityDescriptor(bucket[right]) + }); + if (pairs.length >= 256) return pairs; + } + } + } + return pairs; +} + +async function collectIdentityHints(items, mode, layaClient) { + if (mode === 'algo' || !layaClient) return new Map(); + const pairs = identityCandidatePairs(items); + const hints = new Map(); + for (let index = 0; index < pairs.length; index += 64) { + const results = await layaClient.identityHintBatch(pairs.slice(index, index + 64)); + results.forEach(result => hints.set(result.id, result)); + } + return hints; +} + function catalogInputHash(items) { return stableHash(items .map(item => `${item.id}\0${item.name}\0${item.category_id || ''}`) @@ -125,13 +185,24 @@ function markProjectionBuilding(database = null) { return projectionState(database); } +function loadProjectionItems(database) { + return database.prepare(` + SELECT p.id, p.source_id, p.item_id, p.name, p.category_id, p.data, + c.name AS category_name + FROM playlist_items p + LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id + WHERE p.type = 'live' + ORDER BY p.id + `).all().map(item => { + const data = parseData(item.data); + return { ...item, tvg_id: data.tvgId || data.tvg_id || null }; + }); +} + function rebuild(database = null, options = {}) { database ||= defaultDatabase(); - const items = database.prepare(` - SELECT id, source_id, item_id, name, category_id - FROM playlist_items WHERE type = 'live' - `).all(); - const groups = projectCatalog(items); + const items = options.items || loadProjectionItems(database); + const groups = projectCatalog(items, { mode: options.mode, identityHints: options.identityHints }); const inputHash = catalogInputHash(items); if (!options.stateAlreadyBuilding) markProjectionBuilding(database); @@ -178,6 +249,23 @@ function rebuild(database = null, options = {}) { } } +async function rebuildWithSignals(database = null, options = {}) { + database ||= defaultDatabase(); + const items = loadProjectionItems(database); + const configured = options.mode || (await require('../db').settings.get()).languageDetectionMode || 'algo'; + const mode = ['algo', 'hybrid', 'laya'].includes(configured) ? configured : 'algo'; + let identityHints = new Map(); + if (mode !== 'algo') { + try { + const layaClient = options.layaClient || new (require('./layaService').LayaService)(); + identityHints = await collectIdentityHints(items, mode, layaClient); + } catch (error) { + console.warn('[LiveProjection] Laya identity hints unavailable; continuing deterministically'); + } + } + return rebuild(database, { ...options, items, mode, identityHints }); +} + let activeRebuild = null; let activeState = null; @@ -185,7 +273,7 @@ function startRebuild(database = null, options = {}) { if (activeRebuild) return { alreadyRunning: true, state: activeState, promise: activeRebuild }; database ||= defaultDatabase(); const mark = options.mark || markProjectionBuilding; - const runner = options.runner || rebuild; + const runner = options.runner || rebuildWithSignals; const schedule = options.schedule || setImmediate; activeState = mark(database); activeRebuild = new Promise((resolve, reject) => { @@ -286,8 +374,10 @@ function serializeChannels(rows, includeHidden = false) { module.exports = { catalogInputHash, + collectIdentityHints, displayNameForFingerprint, getChannels, + identityCandidatePairs, normalizeChannelName, markProjectionBuilding, projectCatalog, @@ -295,6 +385,8 @@ module.exports = { qualityForName, queueRebuild, rebuild, + rebuildWithSignals, + safeIdentityDescriptor, serializeChannels, startRebuild }; diff --git a/test/layaService.test.js b/test/layaService.test.js index 16f7906..d77e324 100644 --- a/test/layaService.test.js +++ b/test/layaService.test.js @@ -119,3 +119,24 @@ test('batch validation rejects oversized item counts', () => { error => error instanceof LayaRequestError && error.statusCode === 413 ); }); + +test('identity hints send only sanitized channel metadata to Laya', async () => { + let requestBody; + const service = new LayaService({ + cache: null, + fetch: async (url, options) => { + requestBody = JSON.parse(options.body); + return { + ok: true, + json: async () => ({ results: [{ answers: { sameIdentity: { noul: 0.9, confidence: 0.8 } } }] }) + }; + } + }); + await service.identityHintBatch([{ + id: 'pair', + left: { name: 'Sports http://user:pass@example.test/live', group: 'token=secret', source: 'Provider', tvgId: 'one', streamUrl: 'forbidden' }, + right: { name: 'Sports', group: 'Group', source: 'Provider', tvgId: 'two', password: 'forbidden' } + }]); + assert.deepEqual(Object.keys(requestBody.states[0].left), ['name', 'group', 'source', 'tvgId']); + assert.doesNotMatch(JSON.stringify(requestBody), /example\.test|secret|forbidden|user:pass/); +}); diff --git a/test/liveGroupingScoring.test.js b/test/liveGroupingScoring.test.js new file mode 100644 index 0000000..dee2b92 --- /dev/null +++ b/test/liveGroupingScoring.test.js @@ -0,0 +1,112 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { + MODE_WEIGHTS, + hardBlock, + scoreCandidate +} = require('../server/services/liveGroupingScoring'); +const { + collectIdentityHints, + normalizeChannelName, + projectCatalog +} = require('../server/services/liveProjectionService'); + +const channel = (id, name, extra = {}) => ({ + id, + source_id: 1, + item_id: id, + name, + category_id: 'sports', + fingerprint: normalizeChannelName(name), + ...extra +}); + +test('weights match the validated architecture and remain bounded', () => { + assert.deepEqual(MODE_WEIGHTS.algo, { algorithm: 80, structure: 15, metadata: 5, laya: 0 }); + assert.deepEqual(MODE_WEIGHTS.hybrid, { algorithm: 70, structure: 15, metadata: 5, laya: 10 }); + assert.deepEqual(MODE_WEIGHTS.laya, { algorithm: 55, structure: 15, metadata: 5, laya: 25 }); +}); + +test('exact algorithm match merges without Laya', () => { + const left = channel('a', 'beIN Sports 1 HD'); + const right = channel('b', 'beIN Sports 1 4K'); + assert.equal(scoreCandidate(left, right, { mode: 'algo' }).matched, true); +}); + +test('exact algorithm match survives a contradictory Laya opinion', () => { + const left = channel('a', 'beIN Sports 1 HD'); + const right = channel('b', 'beIN Sports 1 4K'); + const score = scoreCandidate(left, right, { + mode: 'hybrid', + layaHint: { sameIdentityProbability: 0, confidence: 1, number: '2' } + }); + assert.equal(score.matched, true); + assert.equal(score.signals.laya, 0); +}); + +test('different channel numbers block even when Laya proposes a merge', () => { + const left = channel('a', 'beIN Sports 1'); + const right = channel('b', 'beIN Sports 2'); + const score = scoreCandidate(left, right, { + mode: 'laya', layaHint: { sameIdentityProbability: 1, confidence: 1 } + }); + assert.equal(score.matched, false); + assert.equal(score.blocker, 'number_mismatch'); +}); + +test('protected max token blocks beIN Sports Max 10 from beIN Sports 1', () => { + assert.equal(hardBlock(channel('a', 'beIN Sports Max 10'), channel('b', 'beIN Sports 1')), 'protected_token_mismatch'); + assert.equal(projectCatalog([ + channel('a', 'beIN Sports Max 10'), channel('b', 'beIN Sports 1') + ], { mode: 'laya' }).length, 2); +}); + +test('different strong families are blocked before weighted scoring', () => { + assert.equal(hardBlock(channel('a', 'beIN Sports 1'), channel('b', 'Sky Sports 1')), 'strong_family_mismatch'); +}); + +test('low-confidence Laya hint contributes zero', () => { + const score = scoreCandidate( + channel('a', 'beIN Sports 1 Event'), + channel('b', 'beIN Sports 1'), + { mode: 'hybrid', layaHint: { sameIdentityProbability: 1, confidence: 0.64 } } + ); + assert.equal(score.signals.laya, 0); + assert.equal(score.matched, false); +}); + +test('No Laya mode never calls the identity client', async () => { + let calls = 0; + const hints = await collectIdentityHints([ + channel('a', 'beIN Sports 1 Event'), channel('b', 'beIN Sports 1') + ], 'algo', { identityHintBatch: async () => { calls++; return []; } }); + assert.equal(calls, 0); + assert.equal(hints.size, 0); +}); + +test('Laya only calls Laya but still respects blockers', async () => { + let calls = 0; + const items = [channel('a', 'beIN Sports 1 Event'), channel('b', 'beIN Sports 1')]; + const hints = await collectIdentityHints(items, 'laya', { + identityHintBatch: async pairs => { + calls++; + return pairs.map(pair => ({ id: pair.id, sameIdentityProbability: 1, confidence: 1 })); + } + }); + assert.equal(calls, 1); + assert.equal(projectCatalog(items, { mode: 'laya', identityHints: hints }).length, 1); + assert.equal(projectCatalog([ + channel('1', 'beIN Sports 1'), channel('2', 'beIN Sports 2') + ], { mode: 'laya', identityHints: hints }).length, 2); +}); + +test('hybrid mode caps the positive Laya contribution at weight 10', () => { + const score = scoreCandidate( + channel('a', 'beIN Sports 1 Event'), + channel('b', 'beIN Sports 1'), + { mode: 'hybrid', layaHint: { sameIdentityProbability: 1, confidence: 5 } } + ); + assert.equal(score.signals.laya, 1); + assert.equal(score.total, 72); + assert.equal(score.matched, true); +});