Merge branch 'feature/t16-laya-weighted-signal' into develop (#16)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
blomios
2026-10-02 16:42:01 +02:00
5 changed files with 393 additions and 11 deletions

View File

@ -103,6 +103,22 @@ function initialDownloadState() {
};
}
function sanitizeIdentityText(value, maxLength) {
return String(value || '')
.replace(/https?:\/\/\S+/gi, '[redacted]')
.replace(/\b(?:password|passwd|token|username|user)\s*[=:]\s*\S+/gi, '[redacted]')
.slice(0, maxLength);
}
function sanitizeIdentityDescriptor(value) {
return {
name: sanitizeIdentityText(value?.name, 300),
group: sanitizeIdentityText(value?.group, 200),
source: sanitizeIdentityText(value?.source, 120),
tvgId: sanitizeIdentityText(value?.tvgId, 160)
};
}
class LayaService {
constructor(options = {}) {
this.baseUrl = options.baseUrl || process.env.LAYA_BASE_URL || 'http://laya:8000';
@ -300,6 +316,41 @@ class LayaService {
}
return { runtimeAvailable: true, model, results };
}
async identityHintBatch(pairs) {
if (!Array.isArray(pairs) || pairs.length === 0 || pairs.length > MAX_BATCH_ITEMS) {
throw new LayaRequestError(`Identity hint batch must contain 1 to ${MAX_BATCH_ITEMS} pairs`, 400);
}
const response = await this.request('/v1/systemone/batch', {
method: 'POST',
body: JSON.stringify({
states: pairs.map(pair => ({
left: sanitizeIdentityDescriptor(pair.left),
right: sanitizeIdentityDescriptor(pair.right)
})),
model: 'multilingual',
questions: {
sameIdentity: {
type: 'noul',
instructions: 'Do these two TV channel descriptors identify the same numbered channel?'
}
}
})
});
if (!response || !Array.isArray(response.results) || response.results.length !== pairs.length) {
throw new LayaRequestError('Laya runtime returned an invalid identity batch response', 502);
}
return pairs.map((pair, index) => {
const answer = response.results[index]?.answers?.sameIdentity || {};
return {
id: pair.id,
sameIdentityProbability: Number(answer.noul) || 0,
confidence: Number(answer.answer_confidence ?? answer.confidence) || 0,
family: answer.family || null,
number: answer.number || null
};
});
}
}
module.exports = {
@ -310,5 +361,6 @@ module.exports = {
clearDirectoryContents,
directorySize,
initialDownloadState,
sanitizeIdentityDescriptor,
validateBatchPayload
};

View File

@ -0,0 +1,105 @@
const MODE_WEIGHTS = Object.freeze({
algo: Object.freeze({ algorithm: 80, structure: 15, metadata: 5, laya: 0 }),
hybrid: Object.freeze({ algorithm: 70, structure: 15, metadata: 5, laya: 10 }),
laya: Object.freeze({ algorithm: 55, structure: 15, metadata: 5, laya: 25 })
});
const PROTECTED_TOKENS = new Set(['max', 'news', 'premium', 'extra', 'xtra', 'kids', 'movies']);
const STRONG_FAMILIES = ['bein sports', 'sky sports', 'canal plus', 'eurosport', 'espn', 'dazn'];
const LAYA_CONFIDENCE_FLOOR = 0.65;
const MERGE_THRESHOLD = 70;
function tokens(value) {
return String(value || '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
}
function strongFamily(value) {
const text = tokens(value).join(' ');
return STRONG_FAMILIES.find(family => text.includes(family)) || null;
}
function identityNumber(value, family = strongFamily(value)) {
const list = tokens(value);
let start = 0;
if (family) {
const familyTokens = family.split(' ');
for (let index = 0; index <= list.length - familyTokens.length; index++) {
if (familyTokens.every((token, offset) => list[index + offset] === token)) {
start = index + familyTokens.length;
break;
}
}
}
return list.slice(start).find(token => /^\d+$/.test(token)) || null;
}
function identity(value) {
const list = tokens(value);
const family = strongFamily(value);
return {
family,
number: identityNumber(value, family),
protectedTokens: new Set(list.filter(token => PROTECTED_TOKENS.has(token)))
};
}
function sameSet(left, right) {
return left.size === right.size && [...left].every(value => right.has(value));
}
function hardBlock(left, right) {
const a = identity(left.fingerprint || left.name);
const b = identity(right.fingerprint || right.name);
if (!sameSet(a.protectedTokens, b.protectedTokens)) return 'protected_token_mismatch';
if (a.number && b.number && a.number !== b.number) return 'number_mismatch';
if (a.family && b.family && a.family !== b.family) return 'strong_family_mismatch';
return null;
}
function pairKey(left, right) {
return [String(left.id), String(right.id)].sort().join('\0');
}
function layaSignal(hint, leftIdentity, rightIdentity) {
if (!hint || Number(hint.confidence) < LAYA_CONFIDENCE_FLOOR) return 0;
if (hint.number && leftIdentity.number && hint.number !== leftIdentity.number) return 0;
if (hint.number && rightIdentity.number && hint.number !== rightIdentity.number) return 0;
if (hint.family && leftIdentity.family && hint.family !== leftIdentity.family) return 0;
if (hint.family && rightIdentity.family && hint.family !== rightIdentity.family) return 0;
return Number(hint.sameIdentityProbability) >= 0.5
? Math.min(1, Math.max(0, Number(hint.confidence)))
: 0;
}
function scoreCandidate(left, right, options = {}) {
const mode = MODE_WEIGHTS[options.mode] ? options.mode : 'algo';
const weights = MODE_WEIGHTS[mode];
const blocker = hardBlock(left, right);
if (blocker) return { matched: false, blocker, total: 0, signals: { algorithm: 0, structure: 0, metadata: 0, laya: 0 } };
const leftIdentity = identity(left.fingerprint || left.name);
const rightIdentity = identity(right.fingerprint || right.name);
const exact = left.fingerprint === right.fingerprint;
const sameStructure = Boolean(leftIdentity.family && leftIdentity.family === rightIdentity.family &&
leftIdentity.number && leftIdentity.number === rightIdentity.number);
const algorithm = exact ? 1 : (mode !== 'algo' && sameStructure ? 0.6 : 0);
const structure = sameStructure ? 1 : 0;
const metadata = Boolean(
(left.category_id && left.category_id === right.category_id) ||
(left.tvg_id && left.tvg_id === right.tvg_id)
) ? 1 : 0;
const laya = exact ? 0 : layaSignal(options.layaHint, leftIdentity, rightIdentity);
const total = algorithm * weights.algorithm + structure * weights.structure +
metadata * weights.metadata + laya * weights.laya;
return { matched: total >= MERGE_THRESHOLD, blocker: null, total, signals: { algorithm, structure, metadata, laya } };
}
module.exports = {
LAYA_CONFIDENCE_FLOOR,
MERGE_THRESHOLD,
MODE_WEIGHTS,
PROTECTED_TOKENS,
hardBlock,
identity,
pairKey,
scoreCandidate
};

View File

@ -1,4 +1,5 @@
const crypto = require('crypto');
const { pairKey, scoreCandidate, identity } = require('./liveGroupingScoring');
const QUALITY_TOKENS = new Set([
'4k', 'uhd', 'fhd', 'fullhd', 'hd', 'sd', 'hevc', 'h265', 'h264',
@ -62,8 +63,8 @@ function stableHash(value) {
return crypto.createHash('sha256').update(value).digest('hex');
}
function projectCatalog(items) {
const buckets = new Map();
function projectCatalog(items, options = {}) {
const candidateBuckets = new Map();
for (const item of items) {
const fallback = `channel-${item.source_id}:${item.item_id}`;
const fingerprint = normalizeChannelName(item.name) || fallback;
@ -75,11 +76,22 @@ function projectCatalog(items) {
qualityLabel: quality.label,
qualityRank: quality.rank
};
if (!buckets.has(fingerprint)) buckets.set(fingerprint, []);
buckets.get(fingerprint).push(projected);
const itemIdentity = identity(fingerprint);
const candidateKey = itemIdentity.family && itemIdentity.number
? `${itemIdentity.family}:${itemIdentity.number}`
: fingerprint;
if (!candidateBuckets.has(candidateKey)) candidateBuckets.set(candidateKey, []);
const groups = candidateBuckets.get(candidateKey);
const matchingGroup = groups.find(group => scoreCandidate(group[0], projected, {
mode: options.mode || 'algo',
layaHint: options.identityHints?.get(pairKey(group[0], projected))
}).matched);
if (matchingGroup) matchingGroup.push(projected);
else groups.push([projected]);
}
return Array.from(buckets, ([fingerprint, variants]) => {
return Array.from(candidateBuckets.values()).flat().map(variants => {
const fingerprint = variants.map(variant => variant.fingerprint).sort((a, b) => a.length - b.length || a.localeCompare(b))[0];
variants.sort((a, b) => b.qualityRank - a.qualityRank || a.source_id - b.source_id || String(a.item_id).localeCompare(String(b.item_id)));
variants.forEach((variant, index) => { variant.variantRank = index; });
return {
@ -92,6 +104,54 @@ function projectCatalog(items) {
}).sort((a, b) => a.displayName.localeCompare(b.displayName));
}
function safeIdentityDescriptor(item) {
return {
name: String(item.name || '').slice(0, 300),
group: String(item.category_name || item.groupTitle || '').slice(0, 200),
source: String(item.source_name || '').slice(0, 120),
tvgId: String(item.tvg_id || '').slice(0, 160)
};
}
function identityCandidatePairs(items) {
const buckets = new Map();
for (const item of items) {
const fingerprint = normalizeChannelName(item.name);
const itemIdentity = identity(fingerprint);
if (!itemIdentity.family || !itemIdentity.number) continue;
const key = `${itemIdentity.family}:${itemIdentity.number}`;
if (!buckets.has(key)) buckets.set(key, []);
buckets.get(key).push({ ...item, fingerprint });
}
const pairs = [];
for (const bucket of buckets.values()) {
for (let left = 0; left < bucket.length; left++) {
for (let right = left + 1; right < bucket.length; right++) {
if (bucket[left].fingerprint === bucket[right].fingerprint) continue;
if (scoreCandidate(bucket[left], bucket[right], { mode: 'algo' }).blocker) continue;
pairs.push({
id: pairKey(bucket[left], bucket[right]),
left: safeIdentityDescriptor(bucket[left]),
right: safeIdentityDescriptor(bucket[right])
});
if (pairs.length >= 256) return pairs;
}
}
}
return pairs;
}
async function collectIdentityHints(items, mode, layaClient) {
if (mode === 'algo' || !layaClient) return new Map();
const pairs = identityCandidatePairs(items);
const hints = new Map();
for (let index = 0; index < pairs.length; index += 64) {
const results = await layaClient.identityHintBatch(pairs.slice(index, index + 64));
results.forEach(result => hints.set(result.id, result));
}
return hints;
}
function catalogInputHash(items) {
return stableHash(items
.map(item => `${item.id}\0${item.name}\0${item.category_id || ''}`)
@ -125,13 +185,24 @@ function markProjectionBuilding(database = null) {
return projectionState(database);
}
function loadProjectionItems(database) {
return database.prepare(`
SELECT p.id, p.source_id, p.item_id, p.name, p.category_id, p.data,
c.name AS category_name
FROM playlist_items p
LEFT JOIN categories c ON c.source_id = p.source_id AND c.type = 'live' AND c.category_id = p.category_id
WHERE p.type = 'live'
ORDER BY p.id
`).all().map(item => {
const data = parseData(item.data);
return { ...item, tvg_id: data.tvgId || data.tvg_id || null };
});
}
function rebuild(database = null, options = {}) {
database ||= defaultDatabase();
const items = database.prepare(`
SELECT id, source_id, item_id, name, category_id
FROM playlist_items WHERE type = 'live'
`).all();
const groups = projectCatalog(items);
const items = options.items || loadProjectionItems(database);
const groups = projectCatalog(items, { mode: options.mode, identityHints: options.identityHints });
const inputHash = catalogInputHash(items);
if (!options.stateAlreadyBuilding) markProjectionBuilding(database);
@ -178,6 +249,23 @@ function rebuild(database = null, options = {}) {
}
}
async function rebuildWithSignals(database = null, options = {}) {
database ||= defaultDatabase();
const items = loadProjectionItems(database);
const configured = options.mode || (await require('../db').settings.get()).languageDetectionMode || 'algo';
const mode = ['algo', 'hybrid', 'laya'].includes(configured) ? configured : 'algo';
let identityHints = new Map();
if (mode !== 'algo') {
try {
const layaClient = options.layaClient || new (require('./layaService').LayaService)();
identityHints = await collectIdentityHints(items, mode, layaClient);
} catch (error) {
console.warn('[LiveProjection] Laya identity hints unavailable; continuing deterministically');
}
}
return rebuild(database, { ...options, items, mode, identityHints });
}
let activeRebuild = null;
let activeState = null;
@ -185,7 +273,7 @@ function startRebuild(database = null, options = {}) {
if (activeRebuild) return { alreadyRunning: true, state: activeState, promise: activeRebuild };
database ||= defaultDatabase();
const mark = options.mark || markProjectionBuilding;
const runner = options.runner || rebuild;
const runner = options.runner || rebuildWithSignals;
const schedule = options.schedule || setImmediate;
activeState = mark(database);
activeRebuild = new Promise((resolve, reject) => {
@ -286,8 +374,10 @@ function serializeChannels(rows, includeHidden = false) {
module.exports = {
catalogInputHash,
collectIdentityHints,
displayNameForFingerprint,
getChannels,
identityCandidatePairs,
normalizeChannelName,
markProjectionBuilding,
projectCatalog,
@ -295,6 +385,8 @@ module.exports = {
qualityForName,
queueRebuild,
rebuild,
rebuildWithSignals,
safeIdentityDescriptor,
serializeChannels,
startRebuild
};

View File

@ -119,3 +119,24 @@ test('batch validation rejects oversized item counts', () => {
error => error instanceof LayaRequestError && error.statusCode === 413
);
});
test('identity hints send only sanitized channel metadata to Laya', async () => {
let requestBody;
const service = new LayaService({
cache: null,
fetch: async (url, options) => {
requestBody = JSON.parse(options.body);
return {
ok: true,
json: async () => ({ results: [{ answers: { sameIdentity: { noul: 0.9, confidence: 0.8 } } }] })
};
}
});
await service.identityHintBatch([{
id: 'pair',
left: { name: 'Sports http://user:pass@example.test/live', group: 'token=secret', source: 'Provider', tvgId: 'one', streamUrl: 'forbidden' },
right: { name: 'Sports', group: 'Group', source: 'Provider', tvgId: 'two', password: 'forbidden' }
}]);
assert.deepEqual(Object.keys(requestBody.states[0].left), ['name', 'group', 'source', 'tvgId']);
assert.doesNotMatch(JSON.stringify(requestBody), /example\.test|secret|forbidden|user:pass/);
});

View File

@ -0,0 +1,112 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const {
MODE_WEIGHTS,
hardBlock,
scoreCandidate
} = require('../server/services/liveGroupingScoring');
const {
collectIdentityHints,
normalizeChannelName,
projectCatalog
} = require('../server/services/liveProjectionService');
const channel = (id, name, extra = {}) => ({
id,
source_id: 1,
item_id: id,
name,
category_id: 'sports',
fingerprint: normalizeChannelName(name),
...extra
});
test('weights match the validated architecture and remain bounded', () => {
assert.deepEqual(MODE_WEIGHTS.algo, { algorithm: 80, structure: 15, metadata: 5, laya: 0 });
assert.deepEqual(MODE_WEIGHTS.hybrid, { algorithm: 70, structure: 15, metadata: 5, laya: 10 });
assert.deepEqual(MODE_WEIGHTS.laya, { algorithm: 55, structure: 15, metadata: 5, laya: 25 });
});
test('exact algorithm match merges without Laya', () => {
const left = channel('a', 'beIN Sports 1 HD');
const right = channel('b', 'beIN Sports 1 4K');
assert.equal(scoreCandidate(left, right, { mode: 'algo' }).matched, true);
});
test('exact algorithm match survives a contradictory Laya opinion', () => {
const left = channel('a', 'beIN Sports 1 HD');
const right = channel('b', 'beIN Sports 1 4K');
const score = scoreCandidate(left, right, {
mode: 'hybrid',
layaHint: { sameIdentityProbability: 0, confidence: 1, number: '2' }
});
assert.equal(score.matched, true);
assert.equal(score.signals.laya, 0);
});
test('different channel numbers block even when Laya proposes a merge', () => {
const left = channel('a', 'beIN Sports 1');
const right = channel('b', 'beIN Sports 2');
const score = scoreCandidate(left, right, {
mode: 'laya', layaHint: { sameIdentityProbability: 1, confidence: 1 }
});
assert.equal(score.matched, false);
assert.equal(score.blocker, 'number_mismatch');
});
test('protected max token blocks beIN Sports Max 10 from beIN Sports 1', () => {
assert.equal(hardBlock(channel('a', 'beIN Sports Max 10'), channel('b', 'beIN Sports 1')), 'protected_token_mismatch');
assert.equal(projectCatalog([
channel('a', 'beIN Sports Max 10'), channel('b', 'beIN Sports 1')
], { mode: 'laya' }).length, 2);
});
test('different strong families are blocked before weighted scoring', () => {
assert.equal(hardBlock(channel('a', 'beIN Sports 1'), channel('b', 'Sky Sports 1')), 'strong_family_mismatch');
});
test('low-confidence Laya hint contributes zero', () => {
const score = scoreCandidate(
channel('a', 'beIN Sports 1 Event'),
channel('b', 'beIN Sports 1'),
{ mode: 'hybrid', layaHint: { sameIdentityProbability: 1, confidence: 0.64 } }
);
assert.equal(score.signals.laya, 0);
assert.equal(score.matched, false);
});
test('No Laya mode never calls the identity client', async () => {
let calls = 0;
const hints = await collectIdentityHints([
channel('a', 'beIN Sports 1 Event'), channel('b', 'beIN Sports 1')
], 'algo', { identityHintBatch: async () => { calls++; return []; } });
assert.equal(calls, 0);
assert.equal(hints.size, 0);
});
test('Laya only calls Laya but still respects blockers', async () => {
let calls = 0;
const items = [channel('a', 'beIN Sports 1 Event'), channel('b', 'beIN Sports 1')];
const hints = await collectIdentityHints(items, 'laya', {
identityHintBatch: async pairs => {
calls++;
return pairs.map(pair => ({ id: pair.id, sameIdentityProbability: 1, confidence: 1 }));
}
});
assert.equal(calls, 1);
assert.equal(projectCatalog(items, { mode: 'laya', identityHints: hints }).length, 1);
assert.equal(projectCatalog([
channel('1', 'beIN Sports 1'), channel('2', 'beIN Sports 2')
], { mode: 'laya', identityHints: hints }).length, 2);
});
test('hybrid mode caps the positive Laya contribution at weight 10', () => {
const score = scoreCandidate(
channel('a', 'beIN Sports 1 Event'),
channel('b', 'beIN Sports 1'),
{ mode: 'hybrid', layaHint: { sameIdentityProbability: 1, confidence: 5 } }
);
assert.equal(score.signals.laya, 1);
assert.equal(score.total, 72);
assert.equal(score.matched, true);
});