Preserve the guide until a complete EPG refresh can activate
Adapted from MikaelKW/nodecast-tv-plus commits c267d98d734753a09b08d267c8ec2c3fa11c33f5, b761690bac7626a3cc5ddf754d96b14a02f506fe, and 5524fdaffed5c6a44ae69474e7eaa2f18d428c75. Preserve every parser batch, decode gzip once, and retain live channel metadata. Co-authored-by: Mikael <mikaelkwessel@gmail.com>
This commit is contained in:
Technomancer702
@ -120,6 +120,7 @@ router.delete('/:id', async (req, res) => {
|
||||
}
|
||||
|
||||
// Cascade delete: Clean up SQLite data for this source
|
||||
syncService.invalidateEpgSource(sourceId);
|
||||
const db = getDb();
|
||||
const deleteCategories = db.prepare('DELETE FROM categories WHERE source_id = ?');
|
||||
const deleteItems = db.prepare('DELETE FROM playlist_items WHERE source_id = ?');
|
||||
|
||||
@ -1,448 +1,179 @@
|
||||
/**
|
||||
* EPG (XMLTV) Parser (Streaming)
|
||||
* Parses XMLTV format EPG data and extracts channel/programme information using streaming XML parser
|
||||
*/
|
||||
|
||||
/** Streaming XMLTV parsing with bounded batches and complete-document validation. */
|
||||
const sax = require('sax');
|
||||
const zlib = require('zlib');
|
||||
const { Readable } = require('stream');
|
||||
const zlib = require('node:zlib');
|
||||
const { Readable } = require('node:stream');
|
||||
const { pipeline } = require('node:stream/promises');
|
||||
const { StringDecoder } = require('node:string_decoder');
|
||||
const { parseXmltvDate } = require('./xmltvDate');
|
||||
|
||||
/**
|
||||
* Parse XMLTV date format (YYYYMMDDHHmmss +ZZZZ)
|
||||
* @param {string} dateStr - XMLTV format date string
|
||||
* @returns {Date}
|
||||
*/
|
||||
function parseXmltvDate(dateStr) {
|
||||
if (!dateStr) return null;
|
||||
async function* parseStreaming(input, batchSize = 1000) {
|
||||
if (!Number.isInteger(batchSize) || batchSize < 1) throw new Error('Invalid EPG batch size');
|
||||
const stream = typeof input === 'string' ? Readable.from([input]) : input;
|
||||
const parser = sax.parser(true, { trim: true, normalize: true });
|
||||
const decoder = new StringDecoder('utf8');
|
||||
let channels = [], programmes = [], skippedProgrammes = 0;
|
||||
const ready = [];
|
||||
let currentObject = null, objectTag = null, textBuffer = '';
|
||||
let validTimestamps = true, error = null, depth = 0;
|
||||
let sawTvRoot = false, closedTvRoot = false;
|
||||
|
||||
// Format: 20231225120000 +0000
|
||||
const match = dateStr.match(/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})\s*([+-]\d{4})?$/);
|
||||
if (!match) {
|
||||
// Try ISO format fallback
|
||||
return new Date(dateStr);
|
||||
}
|
||||
|
||||
const [, year, month, day, hour, minute, second, tz] = match;
|
||||
let isoStr = `${year}-${month}-${day}T${hour}:${minute}:${second}`;
|
||||
|
||||
if (tz) {
|
||||
const tzHours = tz.substring(0, 3);
|
||||
const tzMins = tz.substring(3);
|
||||
isoStr += `${tzHours}:${tzMins}`;
|
||||
} else {
|
||||
isoStr += 'Z';
|
||||
}
|
||||
|
||||
return new Date(isoStr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse XMLTV content (Stream or String)
|
||||
* @param {Readable|string} input - XMLTV content as Stream or String
|
||||
* @returns {Promise<{ channels: Array, programmes: Array }>}
|
||||
*/
|
||||
function parse(input) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const channels = [];
|
||||
const programmes = [];
|
||||
|
||||
const saxStream = sax.createStream(true, { trim: true, normalize: true }); // strict mode
|
||||
|
||||
let currentTag = null;
|
||||
let currentObject = null;
|
||||
let textBuffer = '';
|
||||
|
||||
saxStream.on('error', function (e) {
|
||||
// clear the error
|
||||
this._parser.error = null;
|
||||
this._parser.resume();
|
||||
console.warn('XML Parse Warning:', e.message);
|
||||
});
|
||||
|
||||
saxStream.on('opentag', function (node) {
|
||||
currentTag = node.name;
|
||||
const attr = node.attributes;
|
||||
|
||||
if (currentTag === 'channel') {
|
||||
currentObject = {
|
||||
id: attr.id,
|
||||
name: null, // Will be populated by display-name tag
|
||||
icon: null,
|
||||
url: null
|
||||
};
|
||||
} else if (currentTag === 'programme') {
|
||||
currentObject = {
|
||||
channelId: attr.channel,
|
||||
start: parseXmltvDate(attr.start),
|
||||
stop: parseXmltvDate(attr.stop),
|
||||
title: null,
|
||||
subtitle: null,
|
||||
description: null,
|
||||
category: [],
|
||||
icon: null,
|
||||
date: null,
|
||||
episodeNum: null
|
||||
};
|
||||
} else if (currentTag === 'icon') {
|
||||
if (currentObject) {
|
||||
currentObject.icon = attr.src;
|
||||
}
|
||||
}
|
||||
textBuffer = '';
|
||||
});
|
||||
|
||||
saxStream.on('text', function (text) {
|
||||
textBuffer += text;
|
||||
});
|
||||
|
||||
saxStream.on('cdata', function (text) {
|
||||
textBuffer += text;
|
||||
});
|
||||
|
||||
saxStream.on('closetag', function (tagName) {
|
||||
if (tagName === 'channel') {
|
||||
if (currentObject) channels.push(currentObject);
|
||||
currentObject = null;
|
||||
} else if (tagName === 'programme') {
|
||||
if (currentObject) programmes.push(currentObject);
|
||||
currentObject = null;
|
||||
} else if (currentObject) {
|
||||
// Handle properties within objects
|
||||
switch (tagName) {
|
||||
case 'display-name': // channel name
|
||||
if (!currentObject.name) currentObject.name = textBuffer;
|
||||
break;
|
||||
case 'url': // channel url
|
||||
currentObject.url = textBuffer;
|
||||
break;
|
||||
case 'title':
|
||||
currentObject.title = textBuffer;
|
||||
break;
|
||||
case 'sub-title':
|
||||
currentObject.subtitle = textBuffer;
|
||||
break;
|
||||
case 'desc':
|
||||
currentObject.description = textBuffer;
|
||||
break;
|
||||
case 'category':
|
||||
if (textBuffer && currentObject.category) currentObject.category.push(textBuffer);
|
||||
break;
|
||||
case 'date':
|
||||
currentObject.date = textBuffer;
|
||||
break;
|
||||
case 'episode-num':
|
||||
// Prefer system "xmltv_ns" or just take text
|
||||
// Complex episode parsing logic can go here if needed
|
||||
currentObject.episodeNum = textBuffer;
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
saxStream.on('end', function () {
|
||||
resolve({ channels, programmes });
|
||||
});
|
||||
|
||||
// Handle input type
|
||||
if (typeof input === 'string') {
|
||||
const inputStream = Readable.from([input]);
|
||||
inputStream.pipe(saxStream);
|
||||
} else {
|
||||
input.pipe(saxStream);
|
||||
const queueBatch = isLast => {
|
||||
ready.push({ channels, programmes, skippedProgrammes, isLast });
|
||||
channels = [];
|
||||
programmes = [];
|
||||
skippedProgrammes = 0;
|
||||
};
|
||||
parser.onerror = e => {
|
||||
error ||= new Error(`Invalid XMLTV document: ${e.message}`);
|
||||
parser.resume();
|
||||
};
|
||||
parser.onopentag = node => {
|
||||
if (depth === 0) {
|
||||
if (node.name !== 'tv' || sawTvRoot) error ||= new Error('Invalid XMLTV document: expected one <tv> root');
|
||||
sawTvRoot = node.name === 'tv';
|
||||
}
|
||||
});
|
||||
depth++;
|
||||
const attr = node.attributes;
|
||||
if (depth === 2 && node.name === 'channel') {
|
||||
objectTag = 'channel';
|
||||
currentObject = { id: attr.id, name: null, icon: null, url: null };
|
||||
} else if (depth === 2 && node.name === 'programme') {
|
||||
objectTag = 'programme';
|
||||
const start = parseXmltvDate(attr.start), stop = parseXmltvDate(attr.stop);
|
||||
validTimestamps = Boolean(attr.channel && start) && (!attr.stop || Boolean(stop));
|
||||
currentObject = { channelId: attr.channel, start, stop, title: null,
|
||||
subtitle: null, description: null, category: [], icon: null, date: null, episodeNum: null };
|
||||
} else if (node.name === 'icon' && currentObject) {
|
||||
currentObject.icon = attr.src;
|
||||
}
|
||||
textBuffer = '';
|
||||
};
|
||||
parser.ontext = text => { textBuffer += text; };
|
||||
parser.oncdata = parser.ontext;
|
||||
parser.onclosetag = tag => {
|
||||
if (depth === 1 && tag === 'tv') closedTvRoot = true;
|
||||
if (depth === 2 && tag === objectTag) {
|
||||
if (objectTag === 'channel' && currentObject.id) {
|
||||
currentObject.name ||= currentObject.id;
|
||||
channels.push(currentObject);
|
||||
} else if (objectTag === 'programme') {
|
||||
if (validTimestamps) programmes.push(currentObject);
|
||||
else skippedProgrammes++;
|
||||
}
|
||||
currentObject = null;
|
||||
objectTag = null;
|
||||
if (programmes.length >= batchSize || channels.length >= batchSize) queueBatch(false);
|
||||
} else if (currentObject) {
|
||||
switch (tag) {
|
||||
case 'display-name': currentObject.name ||= textBuffer; break;
|
||||
case 'url': currentObject.url = textBuffer; break;
|
||||
case 'title': currentObject.title = textBuffer; break;
|
||||
case 'sub-title': currentObject.subtitle = textBuffer; break;
|
||||
case 'desc': currentObject.description = textBuffer; break;
|
||||
case 'category': if (textBuffer && currentObject.category) currentObject.category.push(textBuffer); break;
|
||||
case 'date': currentObject.date = textBuffer; break;
|
||||
case 'episode-num': currentObject.episodeNum = textBuffer; break;
|
||||
}
|
||||
}
|
||||
depth--;
|
||||
};
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
|
||||
// A network chunk can contain many batches. Keep each one and yield
|
||||
// between bounded SAX writes instead of overwriting a pending batch.
|
||||
for (let offset = 0; offset < bytes.length; offset += 16384) {
|
||||
parser.write(decoder.write(bytes.subarray(offset, offset + 16384)));
|
||||
if (error) throw error;
|
||||
while (ready.length) yield ready.shift();
|
||||
}
|
||||
}
|
||||
parser.write(decoder.end()).close();
|
||||
if (error) throw error;
|
||||
if (!sawTvRoot || !closedTvRoot) throw new Error('Invalid XMLTV document: missing complete <tv> root');
|
||||
queueBatch(true);
|
||||
while (ready.length) yield ready.shift();
|
||||
} finally {
|
||||
stream.destroy?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function parse(input) {
|
||||
const channels = [], programmes = [];
|
||||
let skippedProgrammes = 0;
|
||||
for await (const batch of parseStreaming(input)) {
|
||||
channels.push(...batch.channels);
|
||||
programmes.push(...batch.programmes);
|
||||
skippedProgrammes += batch.skippedProgrammes;
|
||||
}
|
||||
return { channels, programmes, skippedProgrammes };
|
||||
}
|
||||
|
||||
async function* fetchAndParseStreaming(url, batchSize = 1000) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`Failed to fetch EPG: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const body = response.body && typeof response.body.pipe === 'function'
|
||||
? response.body : response.body ? Readable.fromWeb(response.body) : Readable.from([]);
|
||||
let source, decoded, completion;
|
||||
try {
|
||||
// Fetch transparently decodes Content-Encoding. Inspect actual bytes
|
||||
// so .gz downloads work without double-decompressing HTTP gzip.
|
||||
const iterator = body[Symbol.asyncIterator]();
|
||||
const prefix = [];
|
||||
let prefixLength = 0;
|
||||
while (prefixLength < 2) {
|
||||
const next = await iterator.next();
|
||||
if (next.done) break;
|
||||
const chunk = Buffer.from(next.value);
|
||||
prefix.push(chunk);
|
||||
prefixLength += chunk.length;
|
||||
}
|
||||
const first = Buffer.concat(prefix);
|
||||
source = Readable.from((async function* () {
|
||||
if (first.length) yield first;
|
||||
for await (const chunk of { [Symbol.asyncIterator]: () => iterator }) yield chunk;
|
||||
})(), { objectMode: false });
|
||||
decoded = source;
|
||||
if (first[0] === 0x1f && first[1] === 0x8b) {
|
||||
decoded = zlib.createGunzip();
|
||||
completion = pipeline(source, decoded);
|
||||
completion.catch(() => {}); // The decoder's async iterator reports failures.
|
||||
}
|
||||
yield* parseStreaming(decoded, batchSize);
|
||||
} finally {
|
||||
decoded?.destroy();
|
||||
source?.destroy();
|
||||
body.destroy();
|
||||
if (completion) await completion.catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchAndParse(url) {
|
||||
const channels = [], programmes = [];
|
||||
let skippedProgrammes = 0;
|
||||
for await (const batch of fetchAndParseStreaming(url)) {
|
||||
channels.push(...batch.channels);
|
||||
programmes.push(...batch.programmes);
|
||||
skippedProgrammes += batch.skippedProgrammes;
|
||||
}
|
||||
return { channels, programmes, skippedProgrammes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get programmes for a specific channel
|
||||
*/
|
||||
function getProgrammesForChannel(programmes, channelId) {
|
||||
return programmes.filter(p => p.channelId === channelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current and upcoming programmes for a channel
|
||||
*/
|
||||
function getCurrentAndUpcoming(programmes, channelId, count = 5) {
|
||||
const now = new Date();
|
||||
const channelProgrammes = getProgrammesForChannel(programmes, channelId);
|
||||
|
||||
// Sort by start time
|
||||
channelProgrammes.sort((a, b) => a.start - b.start);
|
||||
|
||||
// Find current and upcoming
|
||||
const current = channelProgrammes.find(p => p.start <= now && p.stop > now);
|
||||
const upcoming = channelProgrammes
|
||||
.filter(p => p.start > now)
|
||||
.slice(0, count);
|
||||
|
||||
return { current, upcoming };
|
||||
return { current: channelProgrammes.find(p => p.start <= now && p.stop > now),
|
||||
upcoming: channelProgrammes.filter(p => p.start > now).slice(0, count) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and parse XMLTV from URL
|
||||
*/
|
||||
async function fetchAndParse(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch EPG: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
let stream;
|
||||
if (response.body && typeof response.body.pipe === 'function') {
|
||||
stream = response.body;
|
||||
} else if (response.body) {
|
||||
stream = Readable.fromWeb(response.body);
|
||||
} else {
|
||||
stream = Readable.from([]);
|
||||
}
|
||||
|
||||
// Check for GZIP
|
||||
// Note: We can't easily check for magic bytes on a stream without buffering.
|
||||
// We'll rely on response headers or file extension mostly, or try to peek.
|
||||
// For now, let's assume if content-encoding is gzip OR url ends in .gz
|
||||
|
||||
// However, undici/fetch usually handles 'Content-Encoding: gzip' automatically transparently.
|
||||
// We only need to manually gunzip if the server serves it as application/octet-stream but it's actually gzipped,
|
||||
// or if it's a .gz file download.
|
||||
|
||||
// A robust way for streams is checking magic bytes, but that requires peeking.
|
||||
// Simplified approach: try to pipe through gunzip if the URL indicates it.
|
||||
|
||||
const isGzipped = url.endsWith('.gz') || (response.headers.get('content-type') || '').includes('gzip');
|
||||
|
||||
if (isGzipped) {
|
||||
const gunzip = zlib.createGunzip();
|
||||
stream.pipe(gunzip);
|
||||
return parse(gunzip);
|
||||
}
|
||||
|
||||
// In the previous version we read magic bytes.
|
||||
// To support that with streams we'd need a peek stream.
|
||||
// For now let's trust the transparent decompression of fetch or the URL.
|
||||
|
||||
return parse(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming EPG parser that yields batches of programmes (memory-efficient)
|
||||
* Channels are collected and returned with the first batch, then programmes are yielded in batches.
|
||||
*
|
||||
* @param {string} url - XMLTV URL
|
||||
* @param {number} batchSize - Number of programmes per batch (default: 1000)
|
||||
* @yields {{ channels: Array|null, programmes: Array, isLast: boolean }}
|
||||
*/
|
||||
async function* fetchAndParseStreaming(url, batchSize = 1000) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch EPG: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
let stream;
|
||||
if (response.body && typeof response.body.pipe === 'function') {
|
||||
stream = response.body;
|
||||
} else if (response.body) {
|
||||
stream = Readable.fromWeb(response.body);
|
||||
} else {
|
||||
stream = Readable.from([]);
|
||||
}
|
||||
|
||||
const isGzipped = url.endsWith('.gz') || (response.headers.get('content-type') || '').includes('gzip');
|
||||
|
||||
if (isGzipped) {
|
||||
const gunzip = zlib.createGunzip();
|
||||
stream.pipe(gunzip);
|
||||
stream = gunzip;
|
||||
}
|
||||
|
||||
// Use async iterator pattern with SAX
|
||||
yield* parseStreaming(stream, batchSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse XMLTV as streaming async generator
|
||||
* @param {Readable} input - XMLTV stream
|
||||
* @param {number} batchSize - Number of programmes per batch
|
||||
* @yields {{ channels: Array|null, programmes: Array, isLast: boolean }}
|
||||
*/
|
||||
async function* parseStreaming(input, batchSize = 1000) {
|
||||
const channels = [];
|
||||
let programmeBatch = [];
|
||||
let channelsYielded = false;
|
||||
|
||||
// We need to convert SAX events to an async iterator
|
||||
// This requires collecting events and yielding when batch is full
|
||||
|
||||
const saxStream = sax.createStream(true, { trim: true, normalize: true });
|
||||
|
||||
let currentTag = null;
|
||||
let currentObject = null;
|
||||
let textBuffer = '';
|
||||
let resolveNext = null;
|
||||
let pendingBatch = null;
|
||||
let ended = false;
|
||||
let error = null;
|
||||
|
||||
saxStream.on('error', function (e) {
|
||||
this._parser.error = null;
|
||||
this._parser.resume();
|
||||
console.warn('XML Parse Warning:', e.message);
|
||||
});
|
||||
|
||||
saxStream.on('opentag', function (node) {
|
||||
currentTag = node.name;
|
||||
const attr = node.attributes;
|
||||
|
||||
if (currentTag === 'channel') {
|
||||
currentObject = {
|
||||
id: attr.id,
|
||||
name: null,
|
||||
icon: null,
|
||||
url: null
|
||||
};
|
||||
} else if (currentTag === 'programme') {
|
||||
currentObject = {
|
||||
channelId: attr.channel,
|
||||
start: parseXmltvDate(attr.start),
|
||||
stop: parseXmltvDate(attr.stop),
|
||||
title: null,
|
||||
subtitle: null,
|
||||
description: null,
|
||||
category: [],
|
||||
icon: null,
|
||||
date: null,
|
||||
episodeNum: null
|
||||
};
|
||||
} else if (currentTag === 'icon') {
|
||||
if (currentObject) {
|
||||
currentObject.icon = attr.src;
|
||||
}
|
||||
}
|
||||
textBuffer = '';
|
||||
});
|
||||
|
||||
saxStream.on('text', function (text) {
|
||||
textBuffer += text;
|
||||
});
|
||||
|
||||
saxStream.on('cdata', function (text) {
|
||||
textBuffer += text;
|
||||
});
|
||||
|
||||
saxStream.on('closetag', function (tagName) {
|
||||
if (tagName === 'channel') {
|
||||
if (currentObject) channels.push(currentObject);
|
||||
currentObject = null;
|
||||
} else if (tagName === 'programme') {
|
||||
if (currentObject) {
|
||||
programmeBatch.push(currentObject);
|
||||
|
||||
// Check if we should yield a batch
|
||||
if (programmeBatch.length >= batchSize) {
|
||||
const batch = {
|
||||
channels: !channelsYielded ? channels : null,
|
||||
programmes: programmeBatch,
|
||||
isLast: false
|
||||
};
|
||||
channelsYielded = true;
|
||||
programmeBatch = [];
|
||||
|
||||
if (resolveNext) {
|
||||
resolveNext(batch);
|
||||
resolveNext = null;
|
||||
} else {
|
||||
pendingBatch = batch;
|
||||
}
|
||||
}
|
||||
}
|
||||
currentObject = null;
|
||||
} else if (currentObject) {
|
||||
switch (tagName) {
|
||||
case 'display-name':
|
||||
if (!currentObject.name) currentObject.name = textBuffer;
|
||||
break;
|
||||
case 'url':
|
||||
currentObject.url = textBuffer;
|
||||
break;
|
||||
case 'title':
|
||||
currentObject.title = textBuffer;
|
||||
break;
|
||||
case 'sub-title':
|
||||
currentObject.subtitle = textBuffer;
|
||||
break;
|
||||
case 'desc':
|
||||
currentObject.description = textBuffer;
|
||||
break;
|
||||
case 'category':
|
||||
if (textBuffer && currentObject.category) currentObject.category.push(textBuffer);
|
||||
break;
|
||||
case 'date':
|
||||
currentObject.date = textBuffer;
|
||||
break;
|
||||
case 'episode-num':
|
||||
currentObject.episodeNum = textBuffer;
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
saxStream.on('end', function () {
|
||||
ended = true;
|
||||
// Yield final batch
|
||||
const batch = {
|
||||
channels: !channelsYielded ? channels : null,
|
||||
programmes: programmeBatch,
|
||||
isLast: true
|
||||
};
|
||||
if (resolveNext) {
|
||||
resolveNext(batch);
|
||||
resolveNext = null;
|
||||
} else {
|
||||
pendingBatch = batch;
|
||||
}
|
||||
});
|
||||
|
||||
saxStream.on('error', function (e) {
|
||||
error = e;
|
||||
if (resolveNext) {
|
||||
resolveNext(null);
|
||||
}
|
||||
});
|
||||
|
||||
// Start piping
|
||||
input.pipe(saxStream);
|
||||
|
||||
// Yield batches as they become available
|
||||
while (!ended || pendingBatch) {
|
||||
if (pendingBatch) {
|
||||
const batch = pendingBatch;
|
||||
pendingBatch = null;
|
||||
yield batch;
|
||||
if (batch.isLast) break;
|
||||
} else if (!ended) {
|
||||
// Wait for next batch
|
||||
const batch = await new Promise(resolve => {
|
||||
resolveNext = resolve;
|
||||
});
|
||||
if (batch) {
|
||||
yield batch;
|
||||
if (batch.isLast) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
parse,
|
||||
parseXmltvDate,
|
||||
fetchAndParse,
|
||||
fetchAndParseStreaming,
|
||||
parseStreaming,
|
||||
getProgrammesForChannel,
|
||||
getCurrentAndUpcoming
|
||||
};
|
||||
|
||||
module.exports = { parse, parseXmltvDate, fetchAndParse, fetchAndParseStreaming,
|
||||
parseStreaming, getProgrammesForChannel, getCurrentAndUpcoming };
|
||||
|
||||
@ -3,9 +3,12 @@ const { sources, settings } = require('../db'); // For source config and setting
|
||||
const xtreamApi = require('./xtreamApi');
|
||||
const m3uParser = require('./m3uParser');
|
||||
const epgParser = require('./epgParser');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
|
||||
// Sync tracking
|
||||
const activeSyncs = new Set(); // sourceId
|
||||
const activeEpgSyncs = new Set();
|
||||
const epgSourceGenerations = new Map();
|
||||
|
||||
class SyncService {
|
||||
constructor() {
|
||||
@ -14,6 +17,11 @@ class SyncService {
|
||||
this._currentInterval = null;
|
||||
}
|
||||
|
||||
invalidateEpgSource(sourceId) {
|
||||
const key = String(sourceId);
|
||||
epgSourceGenerations.set(key, (epgSourceGenerations.get(key) || 0) + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get when the last global sync completed
|
||||
*/
|
||||
@ -389,107 +397,216 @@ class SyncService {
|
||||
* Processes EPG files in batches to avoid OOM on large EPG data
|
||||
*/
|
||||
async syncEpgFromUrl(sourceId, url) {
|
||||
console.log(`[Sync] Fetching EPG from: ${url.substring(0, 60)}...`);
|
||||
console.log(`[Sync] Fetching EPG from: ${new URL(url).origin}`);
|
||||
|
||||
// Temporary memory logging for verification
|
||||
const logMemory = () => {
|
||||
const used = process.memoryUsage();
|
||||
console.log(`[Sync] Memory: ${Math.round(used.heapUsed / 1024 / 1024)}MB heap`);
|
||||
};
|
||||
|
||||
logMemory();
|
||||
|
||||
const db = getDb();
|
||||
let allChannels = [];
|
||||
let totalProgrammes = 0;
|
||||
let batchCount = 0;
|
||||
|
||||
// Clear old programmes first
|
||||
db.prepare('DELETE FROM epg_programs WHERE source_id = ?').run(sourceId);
|
||||
|
||||
const programmeStmt = db.prepare(`
|
||||
INSERT INTO epg_programs (channel_id, source_id, start_time, end_time, title, description, data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const insertProgrammes = db.transaction((progs) => {
|
||||
for (const p of progs) {
|
||||
programmeStmt.run(
|
||||
p.channelId,
|
||||
sourceId,
|
||||
p.start ? p.start.getTime() : 0,
|
||||
p.stop ? p.stop.getTime() : 0,
|
||||
p.title,
|
||||
p.description || p.desc,
|
||||
JSON.stringify(p)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Stream and process in batches (default 1000 programmes per batch)
|
||||
for await (const batch of epgParser.fetchAndParseStreaming(url)) {
|
||||
batchCount++;
|
||||
|
||||
// Collect channels from first batch
|
||||
if (batch.channels) {
|
||||
allChannels = batch.channels;
|
||||
}
|
||||
|
||||
// Save this batch of programmes immediately
|
||||
if (batch.programmes.length > 0) {
|
||||
insertProgrammes(batch.programmes);
|
||||
totalProgrammes += batch.programmes.length;
|
||||
}
|
||||
|
||||
// Log progress every 10 batches
|
||||
if (batchCount % 10 === 0) {
|
||||
console.log(`[Sync] Processed ${totalProgrammes} programmes so far...`);
|
||||
logMemory();
|
||||
}
|
||||
|
||||
// Yield to event loop
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
const sourceKey = String(sourceId);
|
||||
if (activeEpgSyncs.has(sourceKey)) {
|
||||
throw new Error(`EPG refresh already in progress for source ${sourceId}`);
|
||||
}
|
||||
activeEpgSyncs.add(sourceKey);
|
||||
const sourceGeneration = epgSourceGenerations.get(sourceKey) || 0;
|
||||
let cleanupRun = null;
|
||||
let refreshError = null;
|
||||
|
||||
console.log(`[Sync] EPG Parsed: ${allChannels.length} channels, ${totalProgrammes} programmes`);
|
||||
logMemory();
|
||||
try {
|
||||
|
||||
// Save EPG Channels
|
||||
if (allChannels.length > 0) {
|
||||
// Temporary memory logging for verification
|
||||
const logMemory = () => {
|
||||
const used = process.memoryUsage();
|
||||
console.log(`[Sync] Memory: ${Math.round(used.heapUsed / 1024 / 1024)}MB heap`);
|
||||
};
|
||||
|
||||
logMemory();
|
||||
|
||||
const db = getDb();
|
||||
const runId = randomUUID();
|
||||
let totalChannels = 0;
|
||||
let totalProgrammes = 0;
|
||||
let skippedProgrammes = 0;
|
||||
let batchCount = 0;
|
||||
let sawFinalBatch = false;
|
||||
|
||||
db.exec(`
|
||||
CREATE TEMP TABLE IF NOT EXISTS epg_programs_staging (
|
||||
run_id TEXT NOT NULL,
|
||||
source_id INTEGER NOT NULL,
|
||||
channel_id TEXT NOT NULL,
|
||||
start_time INTEGER NOT NULL,
|
||||
end_time INTEGER NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
data JSON
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS temp.idx_epg_programs_staging_run
|
||||
ON epg_programs_staging(run_id, source_id);
|
||||
|
||||
CREATE TEMP TABLE IF NOT EXISTS epg_channels_staging (
|
||||
run_id TEXT NOT NULL,
|
||||
source_id INTEGER NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
name TEXT,
|
||||
stream_icon TEXT,
|
||||
data JSON,
|
||||
PRIMARY KEY (run_id, id)
|
||||
);
|
||||
`);
|
||||
|
||||
const deleteStagedProgrammes = db.prepare(
|
||||
'DELETE FROM temp.epg_programs_staging WHERE run_id = ?'
|
||||
);
|
||||
const deleteStagedChannels = db.prepare(
|
||||
'DELETE FROM temp.epg_channels_staging WHERE run_id = ?'
|
||||
);
|
||||
cleanupRun = () => {
|
||||
deleteStagedProgrammes.run(runId);
|
||||
deleteStagedChannels.run(runId);
|
||||
};
|
||||
|
||||
// A prior interrupted attempt in this process must never be reused.
|
||||
db.prepare('DELETE FROM temp.epg_programs_staging WHERE source_id = ?').run(sourceId);
|
||||
db.prepare('DELETE FROM temp.epg_channels_staging WHERE source_id = ?').run(sourceId);
|
||||
|
||||
const programmeStmt = db.prepare(`
|
||||
INSERT INTO temp.epg_programs_staging (
|
||||
run_id, source_id, channel_id, start_time, end_time, title, description, data
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
const channelStmt = db.prepare(`
|
||||
INSERT INTO playlist_items (
|
||||
id, source_id, item_id, type, name, stream_icon,
|
||||
stream_url, category_id, data
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
INSERT INTO temp.epg_channels_staging (
|
||||
run_id, source_id, id, item_id, name, stream_icon, data
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, id) DO UPDATE SET
|
||||
item_id = excluded.item_id,
|
||||
name = excluded.name,
|
||||
stream_icon = excluded.stream_icon,
|
||||
data = excluded.data
|
||||
`);
|
||||
|
||||
const insertChannels = db.transaction((chanList) => {
|
||||
for (const ch of chanList) {
|
||||
const id = `${sourceId}:${ch.id}`;
|
||||
channelStmt.run(
|
||||
id,
|
||||
const stageProgrammes = db.transaction((programmes) => {
|
||||
for (const programme of programmes) {
|
||||
programmeStmt.run(
|
||||
runId,
|
||||
sourceId,
|
||||
ch.id,
|
||||
'epg_channel',
|
||||
ch.name,
|
||||
ch.icon || null,
|
||||
null,
|
||||
null,
|
||||
JSON.stringify(ch)
|
||||
programme.channelId,
|
||||
programme.start ? programme.start.getTime() : 0,
|
||||
programme.stop ? programme.stop.getTime() : 0,
|
||||
programme.title,
|
||||
programme.description || programme.desc,
|
||||
JSON.stringify(programme)
|
||||
);
|
||||
}
|
||||
});
|
||||
const stageChannels = db.transaction((channels) => {
|
||||
for (const channel of channels) {
|
||||
// EPG channel IDs often overlap Xtream stream IDs. Keep the
|
||||
// staged row in the same dedicated namespace as the active row.
|
||||
const id = `${sourceId}:epg_channel:${channel.id}`;
|
||||
channelStmt.run(
|
||||
runId,
|
||||
sourceId,
|
||||
id,
|
||||
channel.id,
|
||||
channel.name,
|
||||
channel.icon || null,
|
||||
JSON.stringify(channel)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
insertChannels(allChannels);
|
||||
console.log(`[Sync] Saved ${allChannels.length} EPG channels`);
|
||||
}
|
||||
const activateStagedGuide = db.transaction(() => {
|
||||
db.prepare('DELETE FROM epg_programs WHERE source_id = ?').run(sourceId);
|
||||
db.prepare(`
|
||||
INSERT INTO epg_programs (
|
||||
channel_id, source_id, start_time, end_time, title, description, data
|
||||
)
|
||||
SELECT channel_id, source_id, start_time, end_time, title, description, data
|
||||
FROM temp.epg_programs_staging
|
||||
WHERE run_id = ? AND source_id = ?
|
||||
`).run(runId, sourceId);
|
||||
|
||||
console.log(`[Sync] Saved ${totalProgrammes} programmes`);
|
||||
db.prepare(`
|
||||
DELETE FROM playlist_items
|
||||
WHERE source_id = ? AND type = 'epg_channel'
|
||||
`).run(sourceId);
|
||||
db.prepare(`
|
||||
INSERT INTO playlist_items (
|
||||
id, source_id, item_id, type, name, stream_icon,
|
||||
stream_url, category_id, data
|
||||
)
|
||||
SELECT id, source_id, item_id, 'epg_channel', name, stream_icon,
|
||||
NULL, NULL, data
|
||||
FROM temp.epg_channels_staging
|
||||
WHERE run_id = ? AND source_id = ?
|
||||
`).run(runId, sourceId);
|
||||
|
||||
cleanupRun();
|
||||
});
|
||||
|
||||
const sourceAtStart = await sources.getById(sourceId);
|
||||
|
||||
// Stream into isolated staging tables. The active guide remains
|
||||
// readable throughout download and parsing.
|
||||
for await (const batch of epgParser.fetchAndParseStreaming(url)) {
|
||||
batchCount++;
|
||||
skippedProgrammes += batch.skippedProgrammes || 0;
|
||||
|
||||
if (batch.channels !== null && batch.channels !== undefined) {
|
||||
stageChannels(batch.channels);
|
||||
totalChannels += batch.channels.length;
|
||||
}
|
||||
|
||||
if (batch.programmes.length > 0) {
|
||||
stageProgrammes(batch.programmes);
|
||||
totalProgrammes += batch.programmes.length;
|
||||
}
|
||||
|
||||
if (batch.isLast) sawFinalBatch = true;
|
||||
|
||||
if (batchCount % 10 === 0) {
|
||||
console.log(`[Sync] Staged ${totalProgrammes} programmes so far...`);
|
||||
logMemory();
|
||||
}
|
||||
|
||||
// Keep requests responsive between SQLite batches.
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
|
||||
if (!sawFinalBatch) {
|
||||
throw new Error('EPG refresh ended before the XMLTV document completed');
|
||||
}
|
||||
|
||||
// If a real configured source disappeared while its download was
|
||||
// running, do not recreate any of its guide data.
|
||||
const sourceStillExists = !sourceAtStart || await sources.getById(sourceId);
|
||||
const sourceWasInvalidated = (epgSourceGenerations.get(sourceKey) || 0) !== sourceGeneration;
|
||||
if (sourceWasInvalidated || !sourceStillExists) {
|
||||
throw new Error(`EPG source ${sourceId} was removed during refresh`);
|
||||
}
|
||||
|
||||
console.log(`[Sync] EPG Parsed: ${totalChannels} channels, ${totalProgrammes} programmes`);
|
||||
if (skippedProgrammes > 0) {
|
||||
console.warn(`[Sync] Skipped ${skippedProgrammes} programme entries with invalid XMLTV timestamps`);
|
||||
}
|
||||
logMemory();
|
||||
|
||||
// One short transaction makes programmes and channel mapping
|
||||
// visible together. A valid empty feed intentionally clears both.
|
||||
activateStagedGuide();
|
||||
console.log(`[Sync] Activated ${totalChannels} EPG channels and ${totalProgrammes} programmes`);
|
||||
} catch (error) {
|
||||
refreshError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
if (cleanupRun) cleanupRun();
|
||||
} catch (cleanupError) {
|
||||
console.warn('[Sync] Failed to clean EPG staging data:', cleanupError.message);
|
||||
if (!refreshError) throw cleanupError;
|
||||
} finally {
|
||||
activeEpgSyncs.delete(sourceKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
94
server/services/xmltvDate.js
Normal file
94
server/services/xmltvDate.js
Normal file
@ -0,0 +1,94 @@
|
||||
function createValidatedCalendarDate(year, month, day, hour, minute, second) {
|
||||
if (
|
||||
month < 1 || month > 12 ||
|
||||
day < 1 || day > 31 ||
|
||||
hour < 0 || hour > 23 ||
|
||||
minute < 0 || minute > 59 ||
|
||||
second < 0 || second > 59
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// setUTCFullYear avoids Date.UTC's special handling of years 0-99.
|
||||
const calendarDate = new Date(0);
|
||||
calendarDate.setUTCFullYear(year, month - 1, day);
|
||||
calendarDate.setUTCHours(hour, minute, second, 0);
|
||||
|
||||
// Date normalizes impossible calendar values (for example February 30),
|
||||
// so compare every component before accepting it.
|
||||
if (
|
||||
calendarDate.getUTCFullYear() !== year ||
|
||||
calendarDate.getUTCMonth() !== month - 1 ||
|
||||
calendarDate.getUTCDate() !== day ||
|
||||
calendarDate.getUTCHours() !== hour ||
|
||||
calendarDate.getUTCMinutes() !== minute ||
|
||||
calendarDate.getUTCSeconds() !== second
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return calendarDate;
|
||||
}
|
||||
|
||||
function parseTimezoneOffset(timezone) {
|
||||
if (!timezone || timezone.toUpperCase() === 'Z') return 0;
|
||||
|
||||
const normalized = timezone.replace(':', '');
|
||||
const offsetHours = Number(normalized.slice(1, 3));
|
||||
const offsetRemainder = Number(normalized.slice(3, 5));
|
||||
if (offsetHours > 23 || offsetRemainder > 59) return null;
|
||||
|
||||
const total = (offsetHours * 60) + offsetRemainder;
|
||||
return normalized[0] === '-' ? -total : total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an XMLTV date/time. XMLTV permits YYYYMMDDHHmmss or a shorter
|
||||
* initial precision (for example YYYYMMDDHHmm, YYYYMMDDHH, or YYYYMMDD).
|
||||
* Missing components use the earliest value in that precision and UTC is
|
||||
* assumed when no numeric offset is provided.
|
||||
* @param {string} dateStr - XMLTV format date string
|
||||
* @returns {Date|null}
|
||||
*/
|
||||
function parseXmltvDate(dateStr) {
|
||||
if (!dateStr) return null;
|
||||
|
||||
const value = String(dateStr).trim();
|
||||
const match = value.match(/^(\d{4}(?:\d{2}){0,5})\s*(Z|[+-]\d{4})?$/i);
|
||||
if (!match) {
|
||||
// Preserve the existing compatibility fallback for explicit ISO 8601
|
||||
// timestamps without accepting arbitrary host-dependent date strings.
|
||||
const isoMatch = value.match(/^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,3})?)?(Z|[+-]\d{2}:?\d{2})$/i);
|
||||
if (!isoMatch) return null;
|
||||
|
||||
const [, year, month, day, hour, minute, second = '0', timezone] = isoMatch;
|
||||
if (!createValidatedCalendarDate(
|
||||
Number(year),
|
||||
Number(month),
|
||||
Number(day),
|
||||
Number(hour),
|
||||
Number(minute),
|
||||
Number(second)
|
||||
) || parseTimezoneOffset(timezone) === null) return null;
|
||||
|
||||
const isoDate = new Date(value);
|
||||
return Number.isNaN(isoDate.getTime()) ? null : isoDate;
|
||||
}
|
||||
|
||||
const digits = match[1];
|
||||
const timezone = match[2]?.toUpperCase();
|
||||
const year = Number(digits.slice(0, 4));
|
||||
const month = digits.length >= 6 ? Number(digits.slice(4, 6)) : 1;
|
||||
const day = digits.length >= 8 ? Number(digits.slice(6, 8)) : 1;
|
||||
const hour = digits.length >= 10 ? Number(digits.slice(8, 10)) : 0;
|
||||
const minute = digits.length >= 12 ? Number(digits.slice(10, 12)) : 0;
|
||||
const second = digits.length >= 14 ? Number(digits.slice(12, 14)) : 0;
|
||||
|
||||
const calendarDate = createValidatedCalendarDate(year, month, day, hour, minute, second);
|
||||
const offsetMinutes = parseTimezoneOffset(timezone);
|
||||
if (!calendarDate || offsetMinutes === null) return null;
|
||||
|
||||
return new Date(calendarDate.getTime() - (offsetMinutes * 60 * 1000));
|
||||
}
|
||||
|
||||
module.exports = { parseXmltvDate };
|
||||
Reference in New Issue
Block a user