blomios/NodecastV2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve the guide until a complete EPG refresh can activate

Browse Source 
Adapted from MikaelKW/nodecast-tv-plus commits c267d98d734753a09b08d267c8ec2c3fa11c33f5, b761690bac7626a3cc5ddf754d96b14a02f506fe, and 5524fdaffed5c6a44ae69474e7eaa2f18d428c75. Preserve every parser batch, decode gzip once, and retain live channel metadata.

Co-authored-by: Mikael <mikaelkwessel@gmail.com>
This commit is contained in:
Technomancer702
2026-10-12 16:16:48 -07:00
parent 788d43a4cb
commit 4ed22af9e5
5 changed files with 701 additions and 517 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
server/services/syncService.js
View File

@ -3,9 +3,12 @@ const { sources, settings } = require('../db'); // For source config and setting
const xtreamApi = require('./xtreamApi');
const m3uParser = require('./m3uParser');
const epgParser = require('./epgParser');
const { randomUUID } = require('node:crypto');
// Sync tracking
const activeSyncs = new Set(); // sourceId
const activeEpgSyncs = new Set();
const epgSourceGenerations = new Map();
class SyncService {
constructor() {
@ -14,6 +17,11 @@ class SyncService {
this._currentInterval = null;
}
invalidateEpgSource(sourceId) {
const key = String(sourceId);
epgSourceGenerations.set(key, (epgSourceGenerations.get(key) || 0) + 1);
}
/**
* Get when the last global sync completed
*/
@ -389,107 +397,216 @@ class SyncService {
* Processes EPG files in batches to avoid OOM on large EPG data
*/
async syncEpgFromUrl(sourceId, url) {
console.log(`[Sync] Fetching EPG from: ${url.substring(0, 60)}...`);
console.log(`[Sync] Fetching EPG from: ${new URL(url).origin}`);
// Temporary memory logging for verification
const logMemory = () => {
const used = process.memoryUsage();
console.log(`[Sync] Memory: ${Math.round(used.heapUsed / 1024 / 1024)}MB heap`);
};
logMemory();
const db = getDb();
let allChannels = [];
let totalProgrammes = 0;
let batchCount = 0;
// Clear old programmes first
db.prepare('DELETE FROM epg_programs WHERE source_id = ?').run(sourceId);
const programmeStmt = db.prepare(`
INSERT INTO epg_programs (channel_id, source_id, start_time, end_time, title, description, data)
VALUES (?, ?, ?, ?, ?, ?, ?)
`);
const insertProgrammes = db.transaction((progs) => {
for (const p of progs) {
programmeStmt.run(
p.channelId,
sourceId,
p.start ? p.start.getTime() : 0,
p.stop ? p.stop.getTime() : 0,
p.title,
p.description || p.desc,
JSON.stringify(p)
);
}
});
// Stream and process in batches (default 1000 programmes per batch)
for await (const batch of epgParser.fetchAndParseStreaming(url)) {
batchCount++;
// Collect channels from first batch
if (batch.channels) {
allChannels = batch.channels;
}
// Save this batch of programmes immediately
if (batch.programmes.length > 0) {
insertProgrammes(batch.programmes);
totalProgrammes += batch.programmes.length;
}
// Log progress every 10 batches
if (batchCount % 10 === 0) {
console.log(`[Sync] Processed ${totalProgrammes} programmes so far...`);
logMemory();
}
// Yield to event loop
await new Promise(resolve => setImmediate(resolve));
const sourceKey = String(sourceId);
if (activeEpgSyncs.has(sourceKey)) {
throw new Error(`EPG refresh already in progress for source ${sourceId}`);
}
activeEpgSyncs.add(sourceKey);
const sourceGeneration = epgSourceGenerations.get(sourceKey) || 0;
let cleanupRun = null;
let refreshError = null;
console.log(`[Sync] EPG Parsed: ${allChannels.length} channels, ${totalProgrammes} programmes`);
logMemory();
try {
// Save EPG Channels
if (allChannels.length > 0) {
// Temporary memory logging for verification
const logMemory = () => {
const used = process.memoryUsage();
console.log(`[Sync] Memory: ${Math.round(used.heapUsed / 1024 / 1024)}MB heap`);
};
logMemory();
const db = getDb();
const runId = randomUUID();
let totalChannels = 0;
let totalProgrammes = 0;
let skippedProgrammes = 0;
let batchCount = 0;
let sawFinalBatch = false;
db.exec(`
CREATE TEMP TABLE IF NOT EXISTS epg_programs_staging (
run_id TEXT NOT NULL,
source_id INTEGER NOT NULL,
channel_id TEXT NOT NULL,
start_time INTEGER NOT NULL,
end_time INTEGER NOT NULL,
title TEXT,
description TEXT,
data JSON
);
CREATE INDEX IF NOT EXISTS temp.idx_epg_programs_staging_run
ON epg_programs_staging(run_id, source_id);
CREATE TEMP TABLE IF NOT EXISTS epg_channels_staging (
run_id TEXT NOT NULL,
source_id INTEGER NOT NULL,
id TEXT NOT NULL,
item_id TEXT NOT NULL,
name TEXT,
stream_icon TEXT,
data JSON,
PRIMARY KEY (run_id, id)
);
`);
const deleteStagedProgrammes = db.prepare(
'DELETE FROM temp.epg_programs_staging WHERE run_id = ?'
);
const deleteStagedChannels = db.prepare(
'DELETE FROM temp.epg_channels_staging WHERE run_id = ?'
);
cleanupRun = () => {
deleteStagedProgrammes.run(runId);
deleteStagedChannels.run(runId);
};
// A prior interrupted attempt in this process must never be reused.
db.prepare('DELETE FROM temp.epg_programs_staging WHERE source_id = ?').run(sourceId);
db.prepare('DELETE FROM temp.epg_channels_staging WHERE source_id = ?').run(sourceId);
const programmeStmt = db.prepare(`
INSERT INTO temp.epg_programs_staging (
run_id, source_id, channel_id, start_time, end_time, title, description, data
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`);
const channelStmt = db.prepare(`
INSERT INTO playlist_items (
id, source_id, item_id, type, name, stream_icon,
stream_url, category_id, data
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
INSERT INTO temp.epg_channels_staging (
run_id, source_id, id, item_id, name, stream_icon, data
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(run_id, id) DO UPDATE SET
item_id = excluded.item_id,
name = excluded.name,
stream_icon = excluded.stream_icon,
data = excluded.data
`);
const insertChannels = db.transaction((chanList) => {
for (const ch of chanList) {
const id = `${sourceId}:${ch.id}`;
channelStmt.run(
id,
const stageProgrammes = db.transaction((programmes) => {
for (const programme of programmes) {
programmeStmt.run(
runId,
sourceId,
ch.id,
'epg_channel',
ch.name,
ch.icon || null,
null,
null,
JSON.stringify(ch)
programme.channelId,
programme.start ? programme.start.getTime() : 0,
programme.stop ? programme.stop.getTime() : 0,
programme.title,
programme.description || programme.desc,
JSON.stringify(programme)
);
}
});
const stageChannels = db.transaction((channels) => {
for (const channel of channels) {
// EPG channel IDs often overlap Xtream stream IDs. Keep the
// staged row in the same dedicated namespace as the active row.
const id = `${sourceId}:epg_channel:${channel.id}`;
channelStmt.run(
runId,
sourceId,
id,
channel.id,
channel.name,
channel.icon || null,
JSON.stringify(channel)
);
}
});
insertChannels(allChannels);
console.log(`[Sync] Saved ${allChannels.length} EPG channels`);
}
const activateStagedGuide = db.transaction(() => {
db.prepare('DELETE FROM epg_programs WHERE source_id = ?').run(sourceId);
db.prepare(`
INSERT INTO epg_programs (
channel_id, source_id, start_time, end_time, title, description, data
)
SELECT channel_id, source_id, start_time, end_time, title, description, data
FROM temp.epg_programs_staging
WHERE run_id = ? AND source_id = ?
`).run(runId, sourceId);
console.log(`[Sync] Saved ${totalProgrammes} programmes`);
db.prepare(`
DELETE FROM playlist_items
WHERE source_id = ? AND type = 'epg_channel'
`).run(sourceId);
db.prepare(`
INSERT INTO playlist_items (
id, source_id, item_id, type, name, stream_icon,
stream_url, category_id, data
)
SELECT id, source_id, item_id, 'epg_channel', name, stream_icon,
NULL, NULL, data
FROM temp.epg_channels_staging
WHERE run_id = ? AND source_id = ?
`).run(runId, sourceId);
cleanupRun();
});
const sourceAtStart = await sources.getById(sourceId);
// Stream into isolated staging tables. The active guide remains
// readable throughout download and parsing.
for await (const batch of epgParser.fetchAndParseStreaming(url)) {
batchCount++;
skippedProgrammes += batch.skippedProgrammes || 0;
if (batch.channels !== null && batch.channels !== undefined) {
stageChannels(batch.channels);
totalChannels += batch.channels.length;
}
if (batch.programmes.length > 0) {
stageProgrammes(batch.programmes);
totalProgrammes += batch.programmes.length;
}
if (batch.isLast) sawFinalBatch = true;
if (batchCount % 10 === 0) {
console.log(`[Sync] Staged ${totalProgrammes} programmes so far...`);
logMemory();
}
// Keep requests responsive between SQLite batches.
await new Promise(resolve => setImmediate(resolve));
}
if (!sawFinalBatch) {
throw new Error('EPG refresh ended before the XMLTV document completed');
}
// If a real configured source disappeared while its download was
// running, do not recreate any of its guide data.
const sourceStillExists = !sourceAtStart || await sources.getById(sourceId);
const sourceWasInvalidated = (epgSourceGenerations.get(sourceKey) || 0) !== sourceGeneration;
if (sourceWasInvalidated || !sourceStillExists) {
throw new Error(`EPG source ${sourceId} was removed during refresh`);
}
console.log(`[Sync] EPG Parsed: ${totalChannels} channels, ${totalProgrammes} programmes`);
if (skippedProgrammes > 0) {
console.warn(`[Sync] Skipped ${skippedProgrammes} programme entries with invalid XMLTV timestamps`);
}
logMemory();
// One short transaction makes programmes and channel mapping
// visible together. A valid empty feed intentionally clears both.
activateStagedGuide();
console.log(`[Sync] Activated ${totalChannels} EPG channels and ${totalProgrammes} programmes`);
} catch (error) {
refreshError = error;
throw error;
} finally {
try {
if (cleanupRun) cleanupRun();
} catch (cleanupError) {
console.warn('[Sync] Failed to clean EPG staging data:', cleanupError.message);
if (!refreshError) throw cleanupError;
} finally {
activeEpgSyncs.delete(sourceKey);
}
}
}
/**