fix(runtime): cancel de background task immédiat et fiable depuis le workstate (#199, QA verte)
Transforme l'annulation d'une background task en contrat immédiat, observable et non-best-effort: - domain/background_task: états Cancelling/Cancelled/CancelFailed et transitions idempotentes; un kill PTY qui échoue est surfacé en CancelFailed plutôt qu'en succès silencieux. - infrastructure/runner + store: cancel effectif de la task ciblée, reconcile marque Cancelling même sans live handle, et départage CancelFailed des tâches pending au boot. - application/background + workstate + orchestrator/wake: chemin de cancel initié depuis le workstate, projection synchronisée de l'état (pas d'attente d'un cycle ambigu). - frontend: projection workstate de l'état cancelled/cancelling/cancelFailed (cancellingRow/cancelFailedRow/cancelButtons), normalization et services. Tests QA verts: background_task_cancel, background_task_runner cancel_wins_even_if_output_is_still_available, background_task_store reconcile_marks_cancelling_without_live_handle_cancel_failed_and_pending, workstate.test.tsx.
This commit is contained in:
@ -139,11 +139,14 @@ impl CommandBackgroundRunner {
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
// Reap the real exit status (works for a natural exit and a forced kill).
|
||||
let exit_code = pty
|
||||
.kill(&control.pty_handle)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|status| status.code);
|
||||
let exit_code = match outcome {
|
||||
WaitOutcome::Cancelled => None,
|
||||
WaitOutcome::Expired | WaitOutcome::Exited => pty
|
||||
.kill(&control.pty_handle)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|status| status.code),
|
||||
};
|
||||
|
||||
let finished_at_ms = u64::try_from(clock.now_millis().max(0)).unwrap_or(0);
|
||||
let result = match outcome {
|
||||
@ -281,8 +284,12 @@ impl BackgroundTaskRunner for CommandBackgroundRunner {
|
||||
.cloned();
|
||||
// Unknown task ⇒ already terminal or never ours: cancellation is idempotent.
|
||||
if let Some(control) = control {
|
||||
self.pty
|
||||
.kill(&control.pty_handle)
|
||||
.await
|
||||
.map_err(|e| BackgroundTaskPortError::Runner(format!("pty kill failed: {e}")))?;
|
||||
control.cancelled.store(true, Ordering::SeqCst);
|
||||
control.cancel.notify_one();
|
||||
control.cancel.notify_waiters();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -108,7 +108,8 @@ impl FsBackgroundTaskStore {
|
||||
/// Reconciles persisted tasks against live runtime handles at boot.
|
||||
///
|
||||
/// B2 rule: `Running`/`Waiting` tasks missing from `live_task_ids` are marked
|
||||
/// `Failed` with a synthetic restart-loss failure and become delivery-pending.
|
||||
/// `Failed`; `Cancelling` tasks missing from `live_task_ids` are marked
|
||||
/// `CancelFailed`. Both become delivery-pending.
|
||||
/// Already-terminal undelivered tasks are reported as delivery-pending. `Queued`
|
||||
/// tasks are left open.
|
||||
///
|
||||
@ -127,18 +128,13 @@ impl FsBackgroundTaskStore {
|
||||
for (project_id, doc) in &mut state.docs {
|
||||
for task in &mut doc.tasks {
|
||||
match task.state {
|
||||
BackgroundTaskState::Running | BackgroundTaskState::Waiting
|
||||
BackgroundTaskState::Running
|
||||
| BackgroundTaskState::Waiting
|
||||
| BackgroundTaskState::Cancelling
|
||||
if !live.contains(&task.id) =>
|
||||
{
|
||||
let finished_at_ms = now_ms.max(task.updated_at_ms);
|
||||
let result = BackgroundTaskResult::Failure {
|
||||
finished_at_ms,
|
||||
exit_code: None,
|
||||
error: "background task lost its runtime handle during IdeA restart"
|
||||
.into(),
|
||||
stdout_tail: None,
|
||||
stderr_tail: None,
|
||||
};
|
||||
let result = lost_runtime_result(task.state, finished_at_ms);
|
||||
*task = task.complete(result).map_err(invalid_task)?;
|
||||
report.failed_task_ids.push(task.id);
|
||||
report.delivery_pending_task_ids.push(task.id);
|
||||
@ -158,18 +154,13 @@ impl FsBackgroundTaskStore {
|
||||
for task in &mut doc.tasks {
|
||||
if matches!(
|
||||
task.state,
|
||||
BackgroundTaskState::Running | BackgroundTaskState::Waiting
|
||||
BackgroundTaskState::Running
|
||||
| BackgroundTaskState::Waiting
|
||||
| BackgroundTaskState::Cancelling
|
||||
) && !live.contains(&task.id)
|
||||
{
|
||||
let finished_at_ms = now_ms.max(task.updated_at_ms);
|
||||
let result = BackgroundTaskResult::Failure {
|
||||
finished_at_ms,
|
||||
exit_code: None,
|
||||
error: "background task lost its runtime handle during IdeA restart"
|
||||
.into(),
|
||||
stdout_tail: None,
|
||||
stderr_tail: None,
|
||||
};
|
||||
let result = lost_runtime_result(task.state, finished_at_ms);
|
||||
*task = task.complete(result).map_err(invalid_task)?;
|
||||
}
|
||||
}
|
||||
@ -444,3 +435,21 @@ fn io_error(error: std::io::Error) -> BackgroundTaskPortError {
|
||||
fn invalid_task(error: domain::BackgroundTaskError) -> BackgroundTaskPortError {
|
||||
BackgroundTaskPortError::Invalid(error.to_string())
|
||||
}
|
||||
|
||||
fn lost_runtime_result(state: BackgroundTaskState, finished_at_ms: u64) -> BackgroundTaskResult {
|
||||
if state == BackgroundTaskState::Cancelling {
|
||||
BackgroundTaskResult::CancelFailed {
|
||||
finished_at_ms,
|
||||
error: "background task cancellation lost its runtime handle during IdeA restart"
|
||||
.into(),
|
||||
}
|
||||
} else {
|
||||
BackgroundTaskResult::Failure {
|
||||
finished_at_ms,
|
||||
exit_code: None,
|
||||
error: "background task lost its runtime handle during IdeA restart".into(),
|
||||
stdout_tail: None,
|
||||
stderr_tail: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -33,6 +33,7 @@ struct FakePtyState {
|
||||
handle: Option<PtyHandle>,
|
||||
status: Option<ExitStatus>,
|
||||
kills: usize,
|
||||
fail_kill: bool,
|
||||
scrollback: Vec<u8>,
|
||||
}
|
||||
|
||||
@ -57,6 +58,10 @@ impl FakePty {
|
||||
self.state.lock().unwrap().kills
|
||||
}
|
||||
|
||||
fn fail_kill(&self) {
|
||||
self.state.lock().unwrap().fail_kill = true;
|
||||
}
|
||||
|
||||
fn subscribe_count(&self) -> usize {
|
||||
self.subscribe_count.load(Ordering::SeqCst)
|
||||
}
|
||||
@ -105,6 +110,9 @@ impl PtyPort for FakePty {
|
||||
async fn kill(&self, _handle: &PtyHandle) -> Result<ExitStatus, PtyError> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.kills += 1;
|
||||
if state.fail_kill {
|
||||
return Err(PtyError::Io("kill denied".to_owned()));
|
||||
}
|
||||
let status = state.status.get_or_insert(ExitStatus { code: Some(130) });
|
||||
Ok(*status)
|
||||
}
|
||||
@ -314,6 +322,31 @@ async fn deadline_expires_and_kills_when_wait_never_resolves() {
|
||||
assert_eq!(pty.subscribe_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_surfaces_pty_kill_failure_without_completion() {
|
||||
let pty = Arc::new(FakePty::default());
|
||||
pty.fail_kill();
|
||||
let clock = Arc::new(FakeClock::default());
|
||||
let (runner, _completions) = runner_with(Arc::clone(&pty), clock);
|
||||
let task_id = TaskId::from_uuid(id(16));
|
||||
|
||||
runner
|
||||
.spawn(spawn_spec(task_id, None))
|
||||
.await
|
||||
.expect("spawn succeeds");
|
||||
|
||||
let err = runner
|
||||
.cancel(task_id)
|
||||
.await
|
||||
.expect_err("kill failure surfaces");
|
||||
assert!(matches!(err, domain::BackgroundTaskPortError::Runner(_)));
|
||||
assert_eq!(pty.kill_count(), 1);
|
||||
assert!(
|
||||
runner.pty_handle_for(task_id).is_some(),
|
||||
"failed cancel keeps the live handle available for diagnostics/retry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_wins_even_if_output_is_still_available() {
|
||||
let pty = Arc::new(FakePty::default());
|
||||
|
||||
@ -86,6 +86,12 @@ fn running_task(id: u128, project: u128, owner: u128, now_ms: u64) -> Background
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn cancelling_task(id: u128, project: u128, owner: u128, now_ms: u64) -> BackgroundTask {
|
||||
running_task(id, project, owner, now_ms)
|
||||
.transition(BackgroundTaskState::Cancelling, now_ms + 2)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn completed_task(id: u128, project: u128, owner: u128, now_ms: u64) -> BackgroundTask {
|
||||
running_task(id, project, owner, now_ms)
|
||||
.complete(BackgroundTaskResult::Success {
|
||||
@ -337,6 +343,31 @@ async fn reconcile_marks_running_without_live_handle_failed_and_pending() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_marks_cancelling_without_live_handle_cancel_failed_and_pending() {
|
||||
let tmp = TempDir::new();
|
||||
let store = FsBackgroundTaskStore::new(&tmp.project_path());
|
||||
let lost = cancelling_task(1, 10, 100, 1_000);
|
||||
|
||||
store.create(&lost).await.unwrap();
|
||||
|
||||
let report = store.reconcile_boot(&[], 2_000).await.unwrap();
|
||||
|
||||
assert_eq!(report.failed_task_ids, vec![lost.id]);
|
||||
assert_eq!(report.delivery_pending_task_ids, vec![lost.id]);
|
||||
|
||||
let lost_after = store.get(lost.id).await.unwrap().unwrap();
|
||||
assert_eq!(lost_after.state, BackgroundTaskState::CancelFailed);
|
||||
assert!(lost_after.has_pending_completion_delivery());
|
||||
assert!(matches!(
|
||||
lost_after.result,
|
||||
Some(BackgroundTaskResult::CancelFailed {
|
||||
finished_at_ms: 2_000,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_uses_monotonic_timestamp_when_clock_is_behind() {
|
||||
let tmp = TempDir::new();
|
||||
|
||||
Reference in New Issue
Block a user