fix(orchestrator,agent): ask_agent survit au restart de la cible same-project et cross-project — #289 (RestartRendezvousCoordinator stable par RuntimeAgentKey (projectId, agentId) partagé entre le rendez-vous ask_structured et le scheduler AppAgentSelfRestarter: l'ask actif garde verrou de tour, ticket logique et conversation_id, abandonne le drain de l'ancienne session sans no-reply puis capture le Final de continuation de la session relancée (SELF_RESTART_CONTINUATION_PROMPT partagé, un seul tour de continuation); branche delegated du scheduler: pas de preempt/cancel_head/mark_idle locaux, ready + wait_for_handoff, échecs restart propagés explicitement et cancel en cascade sans relance de continuation; watchdog d'inactivité réarmé par l'activité de contrôle restart; corrélation cross-project (requesterProjectId, requesterAgentId, correlationId) conservée jusqu'au résultat post-restart puis supprimée; stop_agent d'une cible en restart annule le rendez-vous au lieu d'un NotFound; le scheduler standalone conserve son comportement historique sans ask actif; QA verte: orchestrator_service 87/87 dont 6 tests restart (final post-restart, nettoyage busy/workstate/registre, corrélation cross-project, failure explicite, cancel cross-project + stop same-project pendant restart), application lib 171/171 dont watchdog réarmé, backend 89+7 ignored, cargo check -p backend, fmt --check; réserves: pas d'E2E AppImage/vrai provider, 7 tests backend ignorés bind socket local; scripts/ hors périmètre; aucun push)

This commit is contained in:
2026-09-08 22:53:52 +02:00
parent db91b00699
commit d6d989187b
9 changed files with 1204 additions and 61 deletions

View File

@ -168,8 +168,9 @@ pub use orchestrator::{
AgentRestartScheduled, AgentSelfRestartScheduler, AgentWakeService, AskLivenessProbe,
ContextGuardUseCases, LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider,
OrchestratorOutcome, OrchestratorService, ProposeContext, ReadContext, ReadMemory,
RecordTurnProvider, WakeSessionProvider, WatchdogOutcome, WriteMemory,
DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW,
RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState, WakeSessionProvider,
WatchdogOutcome, WriteMemory, DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW,
SELF_RESTART_CONTINUATION_PROMPT,
};
pub use permission::{
GetProjectPermissions, GetProjectPermissionsInput, GetProjectPermissionsOutput,

View File

@ -21,6 +21,7 @@ pub use rendezvous::{
pub use service::{
AgentRestartScheduled, AgentSelfRestartScheduler, AskLivenessProbe, ContextGuardUseCases,
LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider, OrchestratorOutcome,
OrchestratorService, RecordTurnProvider,
OrchestratorService, RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState,
SELF_RESTART_CONTINUATION_PROMPT,
};
pub use wake::{AgentWakeService, WakeSessionProvider};

View File

@ -19,7 +19,7 @@ use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::{Duration, Instant};
use tokio::sync::Mutex as AsyncMutex;
use tokio::sync::{watch, Mutex as AsyncMutex};
use domain::conversation::{ConversationParty, ConversationRegistry, SessionRef, WaitForGraph};
use domain::conversation_log::{ConversationTurn, TurnId, TurnRole};
@ -82,6 +82,330 @@ pub trait AgentSelfRestartScheduler: Send + Sync {
) -> Result<AgentRestartScheduled, AppError>;
}
/// Internal continuation prompt used when a self-restart is handed back to the
/// synchronous inter-agent rendezvous that was driving the old runtime.
pub const SELF_RESTART_CONTINUATION_PROMPT: &str =
"IdeA vient de te redémarrer. Reprends immédiatement le travail en cours là où tu l'avais laissé.";
/// Control-plane state of a self-restart affecting an active logical rendezvous.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RestartRendezvousState {
/// No restart currently affects the logical turn.
Idle,
/// The old runtime is being replaced.
Restarting {
/// Stable restart correlation id.
restart_id: String,
},
/// The replacement structured session is available.
Ready {
/// Stable restart correlation id.
restart_id: String,
},
/// Runtime replacement failed.
Failed {
/// Stable restart correlation id.
restart_id: String,
/// Explicit adapter failure.
message: String,
},
/// The owning logical rendezvous was cancelled.
Cancelled {
/// Stable restart correlation id.
restart_id: String,
},
}
struct RestartRendezvousEntry {
active_waiters: usize,
activity: u64,
state: RestartRendezvousState,
tx: watch::Sender<RestartRendezvousState>,
}
/// Stable, runtime-keyed seam shared by the orchestrator and the detached restart
/// adapter. It keeps a logical ask alive while its concrete provider session is
/// replaced.
#[derive(Default)]
pub struct RestartRendezvousCoordinator {
entries: StdMutex<HashMap<RuntimeAgentKey, RestartRendezvousEntry>>,
}
impl RestartRendezvousCoordinator {
/// Creates an empty runtime-keyed coordinator.
#[must_use]
pub fn new() -> Self {
Self::default()
}
fn register(self: &Arc<Self>, key: RuntimeAgentKey) -> RestartRendezvousRegistration {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let entry = entries.entry(key).or_insert_with(|| {
let state = RestartRendezvousState::Idle;
let (tx, _) = watch::channel(state.clone());
RestartRendezvousEntry {
active_waiters: 0,
activity: 0,
state,
tx,
}
});
entry.active_waiters = entry.active_waiters.saturating_add(1);
RestartRendezvousRegistration {
coordinator: Arc::clone(self),
key,
rx: entry.tx.subscribe(),
}
}
/// Announces a restart synchronously. Returns whether an active ask owns the
/// continuation and therefore must receive the post-restart `Final`.
pub fn begin(&self, key: RuntimeAgentKey, restart_id: String) -> bool {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(entry) = entries.get_mut(&key) else {
return false;
};
if entry.active_waiters == 0 {
return false;
}
entry.activity = entry.activity.saturating_add(1);
entry.state = RestartRendezvousState::Restarting { restart_id };
entry.tx.send_replace(entry.state.clone());
true
}
/// Publishes that the replacement structured session can accept continuation.
pub fn ready(&self, key: RuntimeAgentKey, restart_id: &str) {
self.transition(key, restart_id, |restart_id| {
RestartRendezvousState::Ready { restart_id }
});
}
fn resume_taken(&self, key: RuntimeAgentKey, restart_id: &str) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(entry) = entries.get_mut(&key) else {
return;
};
if !matches!(
&entry.state,
RestartRendezvousState::Ready { restart_id: current } if current == restart_id
) {
return;
}
entry.state = RestartRendezvousState::Idle;
entry.activity = entry.activity.saturating_add(1);
entry.tx.send_replace(entry.state.clone());
}
/// Publishes an explicit restart failure to the awaiting logical rendezvous.
pub fn fail(&self, key: RuntimeAgentKey, restart_id: &str, message: String) {
self.transition(key, restart_id, |restart_id| {
RestartRendezvousState::Failed {
restart_id,
message,
}
});
}
/// Cancels the restart-aware continuation for a target being explicitly stopped.
/// Returns whether a restart handoff was active.
pub fn cancel(&self, key: RuntimeAgentKey) -> bool {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(entry) = entries.get_mut(&key) else {
return false;
};
let restart_id = match &entry.state {
RestartRendezvousState::Restarting { restart_id }
| RestartRendezvousState::Ready { restart_id } => restart_id.clone(),
_ => return false,
};
entry.state = RestartRendezvousState::Cancelled { restart_id };
entry.activity = entry.activity.saturating_add(1);
entry.tx.send_replace(entry.state.clone());
true
}
fn transition(
&self,
key: RuntimeAgentKey,
restart_id: &str,
next: impl FnOnce(String) -> RestartRendezvousState,
) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(entry) = entries.get_mut(&key) else {
return;
};
let matches = match &entry.state {
RestartRendezvousState::Restarting {
restart_id: current,
}
| RestartRendezvousState::Ready {
restart_id: current,
} => current == restart_id,
_ => false,
};
if !matches {
return;
}
entry.activity = entry.activity.saturating_add(1);
entry.state = next(restart_id.to_owned());
entry.tx.send_replace(entry.state.clone());
}
#[must_use]
/// Reports whether the owning rendezvous was cancelled during this restart.
pub fn is_cancelled(&self, key: RuntimeAgentKey, restart_id: &str) -> bool {
self.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
.is_some_and(|entry| {
matches!(
&entry.state,
RestartRendezvousState::Cancelled { restart_id: current }
if current == restart_id
)
})
}
/// Waits until the active ask has accepted the replacement session handoff.
pub async fn wait_for_handoff(
&self,
key: RuntimeAgentKey,
restart_id: &str,
) -> Result<(), String> {
let mut rx = {
let entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
entries
.get(&key)
.ok_or_else(|| "restart rendezvous disappeared".to_owned())?
.tx
.subscribe()
};
loop {
let state = rx.borrow_and_update().clone();
match state {
RestartRendezvousState::Idle => return Ok(()),
RestartRendezvousState::Cancelled {
restart_id: current,
} if current == restart_id => {
return Err("restart rendezvous was cancelled".to_owned());
}
RestartRendezvousState::Failed {
restart_id: current,
message,
} if current == restart_id => return Err(message),
_ => {}
}
rx.changed()
.await
.map_err(|_| "restart rendezvous channel closed".to_owned())?;
}
}
/// Releases terminal coordination state after the detached scheduler observed it.
pub fn finish(&self, key: RuntimeAgentKey, restart_id: &str) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let remove = entries.get(&key).is_some_and(|entry| {
entry.active_waiters == 0
&& matches!(
&entry.state,
RestartRendezvousState::Cancelled { restart_id: current }
| RestartRendezvousState::Failed { restart_id: current, .. }
if current == restart_id
)
});
if remove {
entries.remove(&key);
}
}
/// A restarting target is control-plane activity on every watchdog probe, even
/// when provider/transcript activity is temporarily impossible during teardown.
fn activity_token(&self, key: RuntimeAgentKey) -> Option<u64> {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let entry = entries.get_mut(&key)?;
if matches!(
entry.state,
RestartRendezvousState::Restarting { .. } | RestartRendezvousState::Ready { .. }
) {
entry.activity = entry.activity.saturating_add(1);
}
Some(entry.activity)
}
}
struct RestartRendezvousRegistration {
coordinator: Arc<RestartRendezvousCoordinator>,
key: RuntimeAgentKey,
rx: watch::Receiver<RestartRendezvousState>,
}
impl RestartRendezvousRegistration {
fn state(&mut self) -> RestartRendezvousState {
self.rx.borrow_and_update().clone()
}
async fn changed(&mut self) -> Result<(), watch::error::RecvError> {
self.rx.changed().await
}
}
impl Drop for RestartRendezvousRegistration {
fn drop(&mut self) {
let mut entries = self
.coordinator
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(entry) = entries.get_mut(&self.key) else {
return;
};
entry.active_waiters = entry.active_waiters.saturating_sub(1);
if entry.active_waiters == 0 {
let restart_id = match &entry.state {
RestartRendezvousState::Restarting { restart_id }
| RestartRendezvousState::Ready { restart_id } => Some(restart_id.clone()),
_ => None,
};
if let Some(restart_id) = restart_id {
entry.state = RestartRendezvousState::Cancelled { restart_id };
entry.activity = entry.activity.saturating_add(1);
entry.tx.send_replace(entry.state.clone());
} else if matches!(
entry.state,
RestartRendezvousState::Idle | RestartRendezvousState::Failed { .. }
) {
entries.remove(&self.key);
}
}
}
}
/// Default terminal geometry for an orchestrator-launched agent cell. The UI
/// resizes the PTY to the real cell size on attach; these are sane starting rows
/// /cols so the spawn never fails on a zero-sized terminal.
@ -706,6 +1030,8 @@ pub struct OrchestratorService {
spawn_background_command: Option<Arc<SpawnBackgroundCommand>>,
/// Detached self-restart coordinator, wired at the composition root.
self_restart: Option<Arc<dyn AgentSelfRestartScheduler>>,
/// Stable control-plane handoff between an active ask and a detached restart.
restart_rendezvous: Arc<RestartRendezvousCoordinator>,
}
/// Bundle des quatre use cases C7 sous [`domain::fileguard::FileGuard`], injectés
@ -806,6 +1132,7 @@ impl OrchestratorService {
session_limits: None,
spawn_background_command: None,
self_restart: None,
restart_rendezvous: Arc::new(RestartRendezvousCoordinator::new()),
}
}
@ -837,6 +1164,16 @@ impl OrchestratorService {
self
}
#[must_use]
/// Shares the runtime-keyed restart handoff with the detached restart adapter.
pub fn with_restart_rendezvous(
mut self,
coordinator: Arc<RestartRendezvousCoordinator>,
) -> Self {
self.restart_rendezvous = coordinator;
self
}
/// Branche le store des tâches de fond pour tracer les rendez-vous inter-agents
/// comme [`BackgroundTaskKind::HeadlessRendezvous`] (B6).
///
@ -2393,7 +2730,7 @@ impl OrchestratorService {
conversation_id,
requester,
task,
session.as_ref(),
session,
)
.await
}
@ -2419,7 +2756,7 @@ impl OrchestratorService {
conversation_id: domain::conversation::ConversationId,
requester: Option<AgentId>,
task: String,
session: &dyn domain::ports::AgentSession,
session: Arc<dyn domain::ports::AgentSession>,
) -> Result<OrchestratorOutcome, AppError> {
let (input, mailbox) = match (&self.input, &self.mailbox) {
(Some(i), Some(m)) => (i, m),
@ -2524,57 +2861,131 @@ impl OrchestratorService {
ticket: ticket_id,
});
let local_activity = Arc::new(AtomicU64::new(0));
let drain = drain_with_readiness_announcements_and_activity_outcome(
session,
&task,
None,
input.as_ref(),
agent_key,
announcement_publisher,
Some(Arc::clone(&local_activity)),
let mut restart = self.restart_rendezvous.register(agent_key);
let structured = Arc::clone(
self.structured
.as_ref()
.expect("ask_structured requires the structured registry"),
);
let wait_activity = Arc::clone(&local_activity);
// L'attente du rendez-vous structured, rendue comme `Result<String, AppError>`
// pour être enveloppée par le watchdog. Un flux clos sans `Final` correspond
// au no-reply du chemin headless : la cible a rendu la main sans réponse
// exploitable, on expose donc l'erreur métier retryable plutôt qu'un PROCESS.
// The concrete provider session may disappear under us when the target calls
// `idea_restart_self`. The logical rendezvous remains registered by
// RuntimeAgentKey; on the restart control signal we drop only the old drain,
// wait for the replacement session, then capture its continuation `Final`.
let wait = async {
match drain.await {
Ok(TurnOutcome::Completed(content)) => Ok(content),
Ok(TurnOutcome::RateLimited { resets_at_ms }) => {
let conversation_id = session.conversation_id();
let (conversation_id, resets_at_ms) = match (conversation_id, resets_at_ms) {
(Some(conversation_id), resets_at_ms) => {
(Some(conversation_id), resets_at_ms)
let mut current_session = session;
let mut prompt = task.clone();
loop {
let turn = {
let drain = drain_with_readiness_announcements_and_activity_outcome(
current_session.as_ref(),
&prompt,
None,
input.as_ref(),
agent_key,
announcement_publisher.clone(),
Some(Arc::clone(&wait_activity)),
);
tokio::pin!(drain);
tokio::select! {
biased;
changed = restart.changed() => {
changed.map_err(|_| AppError::Process(
format!("restart coordination closed for target {target}")
))?;
None
}
(None, None) => (None, None),
// Reset connu mais conversation moteur absente : ne pas armer
// une reprise non reprenable ; surfacer le fallback humain.
(None, Some(_)) => (None, None),
};
if let (Some(service), Some(structured)) =
(&self.session_limits, &self.structured)
{
if let Some(node_id) =
structured.node_for_agent_in_project(project.id, &agent_id)
{
service.on_rate_limited(
project.id,
agent_id,
node_id,
conversation_id,
resets_at_ms,
);
outcome = &mut drain => Some(outcome),
}
};
let control_active = !matches!(restart.state(), RestartRendezvousState::Idle);
if let Some(outcome) = turn.filter(|_| !control_active) {
match outcome {
Ok(TurnOutcome::Completed(content)) => return Ok(content),
Ok(TurnOutcome::RateLimited { resets_at_ms }) => {
let conversation_id = current_session.conversation_id();
let (conversation_id, resets_at_ms) =
match (conversation_id, resets_at_ms) {
(Some(conversation_id), resets_at_ms) => {
(Some(conversation_id), resets_at_ms)
}
(None, None) => (None, None),
// Reset connu mais conversation moteur absente : ne pas armer
// une reprise non reprenable ; surfacer le fallback humain.
(None, Some(_)) => (None, None),
};
if let (Some(service), Some(structured)) =
(&self.session_limits, &self.structured)
{
if let Some(node_id) =
structured.node_for_agent_in_project(project.id, &agent_id)
{
service.on_rate_limited(
project.id,
agent_id,
node_id,
conversation_id,
resets_at_ms,
);
}
}
return Err(AppError::Process(
"le tour s'est clos en limite de session, sans contenu Final"
.to_owned(),
));
}
Err(err) if structured_no_reply_error(&err) => {
return Err(AppError::TargetReturnedNoReply(target.to_owned()));
}
Err(err) => return Err(AppError::from(err)),
}
}
loop {
match restart.state() {
RestartRendezvousState::Restarting { .. } => {
restart.changed().await.map_err(|_| {
AppError::Process(format!(
"restart coordination closed for target {target}"
))
})?;
}
RestartRendezvousState::Ready { restart_id } => {
current_session = structured
.session_for_agent_in_project(project.id, &agent_id)
.ok_or_else(|| {
AppError::Process(format!(
"restart completed without a live structured session for target {target}"
))
})?;
self.bind_conversation_session(conversation_id, current_session.id());
self.restart_rendezvous.resume_taken(agent_key, &restart_id);
let _ = restart.state();
prompt = SELF_RESTART_CONTINUATION_PROMPT.to_owned();
wait_activity.fetch_add(1, AtomicOrdering::Relaxed);
break;
}
RestartRendezvousState::Failed { message, .. } => {
return Err(AppError::Process(format!(
"restart failed for target {target}: {message}"
)));
}
RestartRendezvousState::Cancelled { .. } => {
return Err(AppError::Process(format!(
"restart cancelled for target {target}"
)));
}
RestartRendezvousState::Idle => {
restart.changed().await.map_err(|_| {
AppError::Process(format!(
"restart coordination closed for target {target}"
))
})?;
}
}
Err(AppError::Process(
"le tour s'est clos en limite de session, sans contenu Final".to_owned(),
))
}
Err(err) if structured_no_reply_error(&err) => {
Err(AppError::TargetReturnedNoReply(target.to_owned()))
}
Err(err) => Err(AppError::from(err)),
}
};
@ -2590,6 +3001,7 @@ impl OrchestratorService {
started,
local: local_activity,
},
Some(agent_key),
)
.await
{
@ -3278,6 +3690,10 @@ impl OrchestratorService {
}
}
let restart_cancelled = self
.restart_rendezvous
.cancel(runtime_key(project, agent_id));
let structured = self
.structured
.clone()
@ -3317,6 +3733,12 @@ impl OrchestratorService {
.await?
.is_empty()
{
if restart_cancelled {
return Ok(OrchestratorOutcome {
detail: format!("cancelled restarting work for agent {label}"),
reply: None,
});
}
return Err(AppError::NotFound(format!(
"running session for agent {label}"
)));
@ -3547,6 +3969,7 @@ impl OrchestratorService {
agent_id: AgentId,
target: &str,
activity: AskWatchdogActivity,
restart_key: Option<RuntimeAgentKey>,
) -> WatchdogOutcome<R> {
let AskWatchdogActivity {
started,
@ -3559,11 +3982,13 @@ impl OrchestratorService {
#[derive(Default)]
struct CompositeActivity {
local: u64,
restart: u64,
transcript: Option<u64>,
token: u64,
}
let probe = self.ask_liveness_probe.clone();
let restart_rendezvous = Arc::clone(&self.restart_rendezvous);
let root = project_root.clone();
let composite = Arc::new(StdMutex::new(CompositeActivity::default()));
let probe_fn = move || {
@ -3571,20 +3996,26 @@ impl OrchestratorService {
let root = root.clone();
let local_activity = Arc::clone(&local_activity);
let composite = Arc::clone(&composite);
let restart_rendezvous = Arc::clone(&restart_rendezvous);
async move {
let transcript = match &probe {
Some(p) => p(root, agent_id).await,
None => None,
};
let local = local_activity.load(AtomicOrdering::Relaxed);
let restart = restart_key
.and_then(|key| restart_rendezvous.activity_token(key))
.unwrap_or(0);
let mut state = composite.lock().unwrap();
let local_delta = local.saturating_sub(state.local);
let restart_delta = restart.saturating_sub(state.restart);
let transcript_delta = match (state.transcript, transcript) {
(Some(previous), Some(current)) => current.saturating_sub(previous),
(None, Some(_)) => 1,
_ => 0,
};
state.local = state.local.max(local);
state.restart = state.restart.max(restart);
if let Some(current) = transcript {
state.transcript =
Some(state.transcript.map_or(current, |old| old.max(current)));
@ -3592,6 +4023,7 @@ impl OrchestratorService {
state.token = state
.token
.saturating_add(local_delta)
.saturating_add(restart_delta)
.saturating_add(transcript_delta);
Some(state.token)
}
@ -4057,4 +4489,34 @@ mod tests {
"pas de cancel_head après disarm"
);
}
#[tokio::test]
async fn restart_control_activity_rearms_inactivity_watchdog() {
let coordinator = Arc::new(RestartRendezvousCoordinator::new());
let key = rkey(9);
let _registration = coordinator.register(key);
assert!(coordinator.begin(key, "restart-watchdog".to_owned()));
let probe = Arc::clone(&coordinator);
let outcome = run_inactivity_watchdog(
async {
tokio::time::sleep(Duration::from_millis(45)).await;
"continued"
},
Duration::from_millis(10),
Duration::from_millis(100),
Instant::now(),
true,
move || {
let probe = Arc::clone(&probe);
async move { probe.activity_token(key) }
},
|_| {},
|_| {},
|_| {},
)
.await;
assert!(matches!(outcome, WatchdogOutcome::Resolved("continued")));
}
}