fix: show live Laya analysis progress (#21)

Track the Laya identity phase separately from catalog projection progress and expose phase-specific counters through the live status API and Settings presentation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-10-05 23:30:35 +02:00
parent 1abc56c90e
commit 45b56d872f
8 changed files with 281 additions and 22 deletions
--- a/public/js/services/liveAnalysisPresentation.js
+++ b/public/js/services/liveAnalysisPresentation.js
@ -6,33 +6,62 @@
    const BUSY_STATUSES = new Set(['queued', 'building', 'running']);
    const FAILED_STATUSES = new Set(['error', 'failed']);

+    function progressValues(status, phase) {
+        if (phase === 'laya_identity') {
+            return {
+                processed: status.layaProcessed ?? status.phaseProcessed ?? status.batchProcessed ?? status.processedBatches,
+                total: status.layaTotal ?? status.phaseTotal ?? status.batchTotal ?? status.totalBatches,
+                prefix: 'AI batch'
+            };
+        }
+        return { processed: status.processed, total: status.total, prefix: '', suffix: ' channels' };
+    }
+
    function present(status = {}) {
        const state = String(status.status || 'idle').toLowerCase();
        const busy = BUSY_STATUSES.has(state);
        const failed = FAILED_STATUSES.has(state);
        const complete = state === 'success';
-        const processed = Number(status.processed);
-        const total = Number(status.total);
-        const determined = busy && status.processed !== null && status.processed !== undefined &&
-            status.total !== null && status.total !== undefined &&
+        const phase = String(status.phase || '').toLowerCase();
+        const progress = progressValues(status, phase);
+        const processed = Number(progress.processed);
+        const total = Number(progress.total);
+        const determined = busy && phase !== 'writing' &&
+            progress.processed !== null && progress.processed !== undefined &&
+            progress.total !== null && progress.total !== undefined &&
            Number.isFinite(processed) && Number.isFinite(total) && total > 0;
        const progressPercent = determined
            ? Math.max(0, Math.min(100, Math.round(processed / total * 100)))
            : null;
        const previouslyCompleted = Boolean(status.completedAt) || Number(status.projectedCount) > 0;
+        const statusLabel = busy
+            ? phase === 'laya_identity'
+                ? 'Analyzing with local AI…'
+                : phase === 'projecting'
+                    ? 'Building channel catalog…'
+                    : phase === 'writing'
+                        ? 'Saving channel catalog…'
+                        : 'Analyzing channels…'
+            : complete ? 'Analysis complete' : failed ? 'Analysis failed' : 'Ready to analyze';
+        const progressText = phase === 'writing' && busy
+            ? 'Finalizing…'
+            : progressPercent === null
+                ? (busy ? 'In progress…' : '')
+                : progress.prefix
+                    ? `${progress.prefix} ${processed} of ${total} · ${progressPercent}%`
+                    : `${processed} of ${total}${progress.suffix || ''} · ${progressPercent}%`;

        return {
            state,
+            phase,
            busy,
            complete,
            failed,
-            statusLabel: busy ? 'Analyzing channels…' : complete ? 'Analysis complete' : failed ? 'Analysis failed' : 'Ready to analyze',
+            statusLabel,
            buttonLabel: failed ? 'Try again' : complete || previouslyCompleted ? 'Analyze again' : 'Analyze channels',
            progressIndeterminate: busy && progressPercent === null,
            progressPercent,
-            progressText: progressPercent === null
-                ? (busy ? 'Analysis in progress…' : '')
-                : `${processed} of ${total} · ${progressPercent}%`,
+            progressText,
            errorDetail: failed && status.error ? String(status.error) : ''
        };
    }
--- a/server/db/liveProjectionSchema.js
+++ b/server/db/liveProjectionSchema.js
@ -14,8 +14,11 @@ const LIVE_PROJECTION_COLUMNS = {
 };

 const CATALOG_PROJECTION_STATE_COLUMNS = {
+    phase: "TEXT NOT NULL DEFAULT 'idle'",
    processed_count: 'INTEGER NOT NULL DEFAULT 0',
-    total_count: 'INTEGER NOT NULL DEFAULT 0'
+    total_count: 'INTEGER NOT NULL DEFAULT 0',
+    laya_processed_count: 'INTEGER NOT NULL DEFAULT 0',
+    laya_total_count: 'INTEGER NOT NULL DEFAULT 0'
 };

 const LIVE_PROJECTION_SCHEMA_SQL = `
@ -40,8 +43,11 @@ const LIVE_PROJECTION_SCHEMA_SQL = `
        completed_at INTEGER,
        input_hash TEXT,
        projected_count INTEGER NOT NULL DEFAULT 0,
+        phase TEXT NOT NULL DEFAULT 'idle',
        processed_count INTEGER NOT NULL DEFAULT 0,
        total_count INTEGER NOT NULL DEFAULT 0,
+        laya_processed_count INTEGER NOT NULL DEFAULT 0,
+        laya_total_count INTEGER NOT NULL DEFAULT 0,
        error TEXT
    );
 `;
--- a/server/routes/live.js
+++ b/server/routes/live.js
@ -13,8 +13,11 @@ function serializeState(state) {
        completedAt: state.completed_at,
        inputHash: state.input_hash,
        projectedCount: state.projected_count,
+        phase: state.phase,
        processed: state.processed_count,
        total: state.total_count,
+        layaProcessed: state.laya_processed_count,
+        layaTotal: state.laya_total_count,
        error: state.error
    };
 }
--- a/server/services/liveProjectionService.js
+++ b/server/services/liveProjectionService.js
@ -141,21 +141,26 @@ function identityCandidatePairs(items) {
    return pairs;
 }

-async function collectIdentityHints(items, mode, layaClient) {
+async function collectIdentityHints(items, mode, layaClient, options = {}) {
    if (mode === 'algo' || !layaClient) return new Map();
    const pairs = identityCandidatePairs(items);
+    const total = Math.ceil(pairs.length / 64);
+    let processed = 0;
    const hints = new Map();
+    options.onProgress?.({ processed, total });
    for (let index = 0; index < pairs.length; index += 64) {
        const results = await layaClient.identityHintBatch(pairs.slice(index, index + 64));
        results.forEach(result => hints.set(result.id, result));
+        processed += 1;
+        options.onProgress?.({ processed, total });
    }
    return hints;
 }

-async function collectOptionalIdentityHints(items, mode, layaClient) {
+async function collectOptionalIdentityHints(items, mode, layaClient, options = {}) {
    if (mode === 'algo') return new Map();
    try {
-        return await collectIdentityHints(items, mode, layaClient);
+        return await collectIdentityHints(items, mode, layaClient, options);
    } catch (error) {
        console.warn('[LiveProjection] Laya identity hints unavailable; continuing deterministically');
        return new Map();
@ -177,20 +182,33 @@ function projectionState(database = null) {
    database ||= defaultDatabase();
    return database.prepare('SELECT * FROM catalog_projection_state WHERE projection = ?').get('live') || {
        projection: 'live', status: 'missing', started_at: null, completed_at: null,
-        input_hash: null, projected_count: 0, processed_count: 0, total_count: 0, error: null
+        input_hash: null, projected_count: 0, phase: 'idle', processed_count: 0, total_count: 0,
+        laya_processed_count: 0, laya_total_count: 0, error: null
    };
 }

+function updateProjectionProgress(database, phase, progress = {}) {
+    database.prepare(`
+        UPDATE catalog_projection_state SET phase = ?,
+            laya_processed_count = COALESCE(?, laya_processed_count),
+            laya_total_count = COALESCE(?, laya_total_count)
+        WHERE projection = 'live'
+    `).run(phase, progress.layaProcessed ?? null, progress.layaTotal ?? null);
+}
+
 function markProjectionBuilding(database = null) {
    database ||= defaultDatabase();
    const total = database.prepare("SELECT COUNT(*) AS count FROM playlist_items WHERE type = 'live'").get().count;
    const startedAt = Date.now();
    database.prepare(`
        INSERT INTO catalog_projection_state
-            (projection, status, started_at, completed_at, input_hash, projected_count, processed_count, total_count, error)
-        VALUES ('live', 'building', ?, NULL, NULL, 0, 0, ?, NULL)
+            (projection, status, started_at, completed_at, input_hash, projected_count, phase,
+             processed_count, total_count, laya_processed_count, laya_total_count, error)
+        VALUES ('live', 'building', ?, NULL, NULL, 0, 'preparing', 0, ?, 0, 0, NULL)
        ON CONFLICT(projection) DO UPDATE SET status = 'building', started_at = excluded.started_at,
-            completed_at = NULL, processed_count = 0, total_count = excluded.total_count, error = NULL
+            completed_at = NULL, phase = 'preparing', processed_count = 0,
+            total_count = excluded.total_count, laya_processed_count = 0,
+            laya_total_count = 0, error = NULL
    `).run(startedAt, total);
    return projectionState(database);
 }
@ -212,9 +230,11 @@ function loadProjectionItems(database) {
 function rebuild(database = null, options = {}) {
    database ||= defaultDatabase();
    const items = options.items || loadProjectionItems(database);
+    if (!options.stateAlreadyBuilding) markProjectionBuilding(database);
+    updateProjectionProgress(database, 'projecting');
    const groups = projectCatalog(items, { mode: options.mode, identityHints: options.identityHints });
    const inputHash = catalogInputHash(items);
-    if (!options.stateAlreadyBuilding) markProjectionBuilding(database);
+    updateProjectionProgress(database, 'writing');

    try {
        const apply = database.transaction(() => {
@ -243,7 +263,7 @@ function rebuild(database = null, options = {}) {
                }
            }
            database.prepare(`
-                UPDATE catalog_projection_state SET status = 'success', completed_at = ?,
+                UPDATE catalog_projection_state SET status = 'success', phase = 'complete', completed_at = ?,
                    input_hash = ?, projected_count = ?, processed_count = ?, total_count = ?, error = NULL
                WHERE projection = 'live'
            `).run(Date.now(), inputHash, groups.length, items.length, items.length);
@ -254,7 +274,7 @@ function rebuild(database = null, options = {}) {
        const diagnosticCode = /^SQLITE_[A-Z_]+$/.test(error.code || '') ? error.code : 'INTERNAL';
        console.error(`[LiveProjection] Rebuild transaction failed (${diagnosticCode}, ${error.name || 'Error'})`);
        database.prepare(`
-            UPDATE catalog_projection_state SET status = 'error', completed_at = ?, error = ?
+            UPDATE catalog_projection_state SET status = 'error', phase = 'failed', completed_at = ?, error = ?
            WHERE projection = 'live'
        `).run(Date.now(), `Live catalog projection failed [${diagnosticCode}]`);
        throw error;
@ -269,7 +289,12 @@ async function rebuildWithSignals(database = null, options = {}) {
    let identityHints = new Map();
    if (mode !== 'algo') {
        const layaClient = options.layaClient || new (require('./layaService').LayaService)();
-        identityHints = await collectOptionalIdentityHints(items, mode, layaClient);
+        identityHints = await collectOptionalIdentityHints(items, mode, layaClient, {
+            onProgress: ({ processed, total }) => updateProjectionProgress(database, 'laya_identity', {
+                layaProcessed: processed,
+                layaTotal: total
+            })
+        });
    }
    return rebuild(database, { ...options, items, mode, identityHints });
 }
--- a/test/liveAnalysisPresentation.test.js
+++ b/test/liveAnalysisPresentation.test.js
@ -3,21 +3,47 @@ const assert = require('node:assert/strict');
 const { layaFallbackMessage, present } = require('../public/js/services/liveAnalysisPresentation.js');

 test('presents determinate channel analysis progress', () => {
-    const view = present({ status: 'building', processed: 25, total: 100 });
+    const view = present({ status: 'building', phase: 'projecting', processed: 25, total: 100 });
    assert.equal(view.busy, true);
+    assert.equal(view.statusLabel, 'Building channel catalog…');
    assert.equal(view.progressIndeterminate, false);
    assert.equal(view.progressPercent, 25);
-    assert.equal(view.progressText, '25 of 100 · 25%');
+    assert.equal(view.progressText, '25 of 100 channels · 25%');
+});
+
+test('presents Laya identity progress using AI batch counters', () => {
+    const view = present({
+        status: 'building',
+        phase: 'laya_identity',
+        layaProcessed: 3,
+        layaTotal: 8,
+        processed: 0,
+        total: 39348
+    });
+    assert.equal(view.statusLabel, 'Analyzing with local AI…');
+    assert.equal(view.progressIndeterminate, false);
+    assert.equal(view.progressPercent, 38);
+    assert.equal(view.progressText, 'AI batch 3 of 8 · 38%');
 });

 test('presents indeterminate progress when totals are unavailable', () => {
    const view = present({ status: 'queued', processed: null, total: null });
    assert.equal(view.busy, true);
    assert.equal(view.progressIndeterminate, true);
+    assert.equal(view.progressPercent, null);
+    assert.equal(view.progressText, 'In progress…');
    assert.equal(view.buttonLabel, 'Analyze channels');
    assert.equal(present({ status: 'running', processed: null, total: 100 }).progressIndeterminate, true);
 });

+test('presents writing as an indeterminate finalization phase', () => {
+    const view = present({ status: 'building', phase: 'writing', processed: 39348, total: 39348 });
+    assert.equal(view.statusLabel, 'Saving channel catalog…');
+    assert.equal(view.progressIndeterminate, true);
+    assert.equal(view.progressPercent, null);
+    assert.equal(view.progressText, 'Finalizing…');
+});
+
 test('presents success and retry states', () => {
    assert.deepEqual(
        { label: present({ status: 'success', projectedCount: 12 }).statusLabel, button: present({ status: 'success' }).buttonLabel },
--- a/test/liveAnalysisUserFlowDom.test.js
+++ b/test/liveAnalysisUserFlowDom.test.js
@ -92,6 +92,43 @@ function projectedChannels() {
    };
 }

+test('Settings renders Laya batch progress and removes misleading values when its total is unknown', () => {
+    const { window } = createDom();
+    const settingsPage = Object.create(window.SettingsPage.prototype);
+    settingsPage.app = { player: { settings: { languageDetectionMode: 'hybrid' } } };
+    settingsPage.layaStatus = { modelPresent: true, runtimeAvailable: true };
+
+    settingsPage.renderLiveAnalysisStatus({
+        status: 'building',
+        phase: 'laya_identity',
+        layaProcessed: 3,
+        layaTotal: 8,
+        processed: 0,
+        total: 39348
+    });
+
+    const track = window.document.getElementById('live-analysis-progress-track');
+    const bar = window.document.getElementById('live-analysis-progress-bar');
+    assert.equal(window.document.getElementById('live-analysis-status').textContent, 'Analyzing with local AI…');
+    assert.equal(window.document.getElementById('live-analysis-progress-text').textContent, 'AI batch 3 of 8 · 38%');
+    assert.equal(track.getAttribute('aria-valuenow'), '38');
+    assert.equal(bar.style.width, '38%');
+
+    settingsPage.renderLiveAnalysisStatus({
+        status: 'building',
+        phase: 'laya_identity',
+        layaProcessed: 0,
+        layaTotal: null,
+        processed: 0,
+        total: 39348
+    });
+
+    assert.equal(window.document.getElementById('live-analysis-progress-text').textContent, 'In progress…');
+    assert.equal(track.classList.contains('indeterminate'), true);
+    assert.equal(track.hasAttribute('aria-valuenow'), false);
+    assert.equal(bar.style.width, '');
+});
+
 test('user analysis click keeps visible channels after Local AI language failure', async () => {
    const { window } = createDom();
    let rebuildClicked = 0;
--- a/test/liveProjectionRoutes.test.js
+++ b/test/liveProjectionRoutes.test.js
@ -36,3 +36,25 @@ test('projection success with live input but zero projected channels is exposed
    assert.equal(state.projectedCount, 0);
    assert.equal(state.total, 3);
 });
+
+test('projection status exposes phase-specific Laya progress separately from catalog progress', () => {
+    const state = serializeState({
+        status: 'building',
+        phase: 'laya_identity',
+        started_at: 100,
+        completed_at: null,
+        input_hash: null,
+        projected_count: 0,
+        processed_count: 0,
+        total_count: 12,
+        laya_processed_count: 1,
+        laya_total_count: 2,
+        error: null
+    });
+
+    assert.equal(state.phase, 'laya_identity');
+    assert.equal(state.processed, 0);
+    assert.equal(state.total, 12);
+    assert.equal(state.layaProcessed, 1);
+    assert.equal(state.layaTotal, 2);
+});
--- a/test/liveProjectionService.test.js
+++ b/test/liveProjectionService.test.js
@ -1,8 +1,12 @@
 const test = require('node:test');
 const assert = require('node:assert/strict');
+const Database = require('better-sqlite3');
 const {
    normalizeChannelName,
    projectCatalog,
+    markProjectionBuilding,
+    projectionState,
+    rebuildWithSignals,
    serializeChannels,
    startRebuild
 } = require('../server/services/liveProjectionService');
@ -20,6 +24,64 @@ const item = (id, name, extra = {}) => ({
    ...extra
 });

+function createProjectionDb() {
+    const db = new Database(':memory:');
+    db.exec(`
+        CREATE TABLE categories (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            category_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            parent_id TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            data JSON
+        );
+        CREATE TABLE playlist_items (
+            id TEXT PRIMARY KEY,
+            source_id INTEGER NOT NULL,
+            item_id TEXT NOT NULL,
+            type TEXT NOT NULL,
+            name TEXT NOT NULL,
+            category_id TEXT,
+            parent_id TEXT,
+            stream_icon TEXT,
+            stream_url TEXT,
+            container_extension TEXT,
+            rating REAL,
+            year TEXT,
+            added_at TEXT,
+            is_hidden INTEGER DEFAULT 0,
+            is_favorite INTEGER DEFAULT 0,
+            data JSON
+        );
+    `);
+    const { ensureColumns } = require('../server/db/liveProjectionSchema');
+    ensureColumns(db, 'playlist_items', LIVE_PROJECTION_COLUMNS);
+    db.exec(LIVE_PROJECTION_SCHEMA_SQL);
+    return db;
+}
+
+function seedProjectionChannel(db, itemId, name) {
+    db.prepare(`
+        INSERT INTO playlist_items
+            (id, source_id, item_id, type, name, category_id, stream_icon, stream_url, data)
+        VALUES (?, 1, ?, 'live', ?, 'sports', '', ?, '{}')
+    `).run(`1:${itemId}`, itemId, name, `https://streams.test/${itemId}.m3u8`);
+}
+
+function waitFor(condition, message) {
+    return new Promise((resolve, reject) => {
+        const started = Date.now();
+        const poll = () => {
+            if (condition()) return resolve();
+            if (Date.now() - started > 1000) return reject(new Error(message));
+            setTimeout(poll, 5);
+        };
+        poll();
+    });
+}
+
 test('projection migration declares grouping and language columns', () => {
    assert.deepEqual(Object.keys(LIVE_PROJECTION_COLUMNS), [
        'logical_channel_id', 'grouping_fingerprint', 'grouping_input_hash',
@ -51,6 +113,55 @@ test('manual rebuild scheduling coalesces concurrent requests', async () => {
    assert.equal(runs, 1);
 });

+test('Laya rebuild progress advances after a successful identity batch while analysis is still running', async () => {
+    const db = createProjectionDb();
+    for (let index = 0; index < 12; index++) {
+        seedProjectionChannel(db, `bein-${index}`, `beIN Sports 1 regional ${index}`);
+    }
+
+    let calls = 0;
+    let secondBatchStarted = false;
+    let releaseSecondBatch;
+    const secondBatchGate = new Promise(resolve => { releaseSecondBatch = resolve; });
+    const layaClient = {
+        identityHintBatch: async pairs => {
+            calls += 1;
+            if (calls === 2) {
+                secondBatchStarted = true;
+                await secondBatchGate;
+            }
+            return pairs.map(pair => ({
+                id: pair.id,
+                sameIdentityProbability: 0.9,
+                confidence: 0.9,
+                family: 'bein sports',
+                number: '1'
+            }));
+        }
+    };
+
+    markProjectionBuilding(db);
+    const rebuild = rebuildWithSignals(db, {
+        mode: 'laya',
+        layaClient,
+        stateAlreadyBuilding: true
+    });
+
+    try {
+        await waitFor(() => secondBatchStarted, 'Expected the second Laya identity batch to start');
+        const state = projectionState(db);
+        assert.equal(state.status, 'building');
+        assert.equal(state.phase, 'laya_identity');
+        assert.equal(state.total_count, 12);
+        assert.equal(state.processed_count, 0);
+        assert.equal(state.laya_processed_count, 1);
+        assert.equal(state.laya_total_count, 2);
+    } finally {
+        releaseSecondBatch();
+        await rebuild;
+    }
+});
+
 test('projects beIN quality variants into one logical channel', () => {
    const groups = projectCatalog([
        item('uhd', '|FR| BEIN SPORTS 1 UHD'),