fix(orchestrator,agent): ask_agent survit au restart de la cible same-project et cross-project — #289 (RestartRendezvousCoordinator stable par RuntimeAgentKey (projectId, agentId) partagé entre le rendez-vous ask_structured et le scheduler AppAgentSelfRestarter: l'ask actif garde verrou de tour, ticket logique et conversation_id, abandonne le drain de l'ancienne session sans no-reply puis capture le Final de continuation de la session relancée (SELF_RESTART_CONTINUATION_PROMPT partagé, un seul tour de continuation); branche delegated du scheduler: pas de preempt/cancel_head/mark_idle locaux, ready + wait_for_handoff, échecs restart propagés explicitement et cancel en cascade sans relance de continuation; watchdog d'inactivité réarmé par l'activité de contrôle restart; corrélation cross-project (requesterProjectId, requesterAgentId, correlationId) conservée jusqu'au résultat post-restart puis supprimée; stop_agent d'une cible en restart annule le rendez-vous au lieu d'un NotFound; le scheduler standalone conserve son comportement historique sans ask actif; QA verte: orchestrator_service 87/87 dont 6 tests restart (final post-restart, nettoyage busy/workstate/registre, corrélation cross-project, failure explicite, cancel cross-project + stop same-project pendant restart), application lib 171/171 dont watchdog réarmé, backend 89+7 ignored, cargo check -p backend, fmt --check; réserves: pas d'E2E AppImage/vrai provider, 7 tests backend ignorés bind socket local; scripts/ hors périmètre; aucun push)
This commit is contained in:
@ -168,8 +168,9 @@ pub use orchestrator::{
|
||||
AgentRestartScheduled, AgentSelfRestartScheduler, AgentWakeService, AskLivenessProbe,
|
||||
ContextGuardUseCases, LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider,
|
||||
OrchestratorOutcome, OrchestratorService, ProposeContext, ReadContext, ReadMemory,
|
||||
RecordTurnProvider, WakeSessionProvider, WatchdogOutcome, WriteMemory,
|
||||
DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW,
|
||||
RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState, WakeSessionProvider,
|
||||
WatchdogOutcome, WriteMemory, DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW,
|
||||
SELF_RESTART_CONTINUATION_PROMPT,
|
||||
};
|
||||
pub use permission::{
|
||||
GetProjectPermissions, GetProjectPermissionsInput, GetProjectPermissionsOutput,
|
||||
|
||||
@ -21,6 +21,7 @@ pub use rendezvous::{
|
||||
pub use service::{
|
||||
AgentRestartScheduled, AgentSelfRestartScheduler, AskLivenessProbe, ContextGuardUseCases,
|
||||
LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider, OrchestratorOutcome,
|
||||
OrchestratorService, RecordTurnProvider,
|
||||
OrchestratorService, RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState,
|
||||
SELF_RESTART_CONTINUATION_PROMPT,
|
||||
};
|
||||
pub use wake::{AgentWakeService, WakeSessionProvider};
|
||||
|
||||
@ -19,7 +19,7 @@ use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
use tokio::sync::{watch, Mutex as AsyncMutex};
|
||||
|
||||
use domain::conversation::{ConversationParty, ConversationRegistry, SessionRef, WaitForGraph};
|
||||
use domain::conversation_log::{ConversationTurn, TurnId, TurnRole};
|
||||
@ -82,6 +82,330 @@ pub trait AgentSelfRestartScheduler: Send + Sync {
|
||||
) -> Result<AgentRestartScheduled, AppError>;
|
||||
}
|
||||
|
||||
/// Internal continuation prompt used when a self-restart is handed back to the
|
||||
/// synchronous inter-agent rendezvous that was driving the old runtime.
|
||||
pub const SELF_RESTART_CONTINUATION_PROMPT: &str =
|
||||
"IdeA vient de te redémarrer. Reprends immédiatement le travail en cours là où tu l'avais laissé.";
|
||||
|
||||
/// Control-plane state of a self-restart affecting an active logical rendezvous.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RestartRendezvousState {
|
||||
/// No restart currently affects the logical turn.
|
||||
Idle,
|
||||
/// The old runtime is being replaced.
|
||||
Restarting {
|
||||
/// Stable restart correlation id.
|
||||
restart_id: String,
|
||||
},
|
||||
/// The replacement structured session is available.
|
||||
Ready {
|
||||
/// Stable restart correlation id.
|
||||
restart_id: String,
|
||||
},
|
||||
/// Runtime replacement failed.
|
||||
Failed {
|
||||
/// Stable restart correlation id.
|
||||
restart_id: String,
|
||||
/// Explicit adapter failure.
|
||||
message: String,
|
||||
},
|
||||
/// The owning logical rendezvous was cancelled.
|
||||
Cancelled {
|
||||
/// Stable restart correlation id.
|
||||
restart_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
struct RestartRendezvousEntry {
|
||||
active_waiters: usize,
|
||||
activity: u64,
|
||||
state: RestartRendezvousState,
|
||||
tx: watch::Sender<RestartRendezvousState>,
|
||||
}
|
||||
|
||||
/// Stable, runtime-keyed seam shared by the orchestrator and the detached restart
|
||||
/// adapter. It keeps a logical ask alive while its concrete provider session is
|
||||
/// replaced.
|
||||
#[derive(Default)]
|
||||
pub struct RestartRendezvousCoordinator {
|
||||
entries: StdMutex<HashMap<RuntimeAgentKey, RestartRendezvousEntry>>,
|
||||
}
|
||||
|
||||
impl RestartRendezvousCoordinator {
|
||||
/// Creates an empty runtime-keyed coordinator.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
fn register(self: &Arc<Self>, key: RuntimeAgentKey) -> RestartRendezvousRegistration {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let entry = entries.entry(key).or_insert_with(|| {
|
||||
let state = RestartRendezvousState::Idle;
|
||||
let (tx, _) = watch::channel(state.clone());
|
||||
RestartRendezvousEntry {
|
||||
active_waiters: 0,
|
||||
activity: 0,
|
||||
state,
|
||||
tx,
|
||||
}
|
||||
});
|
||||
entry.active_waiters = entry.active_waiters.saturating_add(1);
|
||||
RestartRendezvousRegistration {
|
||||
coordinator: Arc::clone(self),
|
||||
key,
|
||||
rx: entry.tx.subscribe(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Announces a restart synchronously. Returns whether an active ask owns the
|
||||
/// continuation and therefore must receive the post-restart `Final`.
|
||||
pub fn begin(&self, key: RuntimeAgentKey, restart_id: String) -> bool {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(entry) = entries.get_mut(&key) else {
|
||||
return false;
|
||||
};
|
||||
if entry.active_waiters == 0 {
|
||||
return false;
|
||||
}
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
entry.state = RestartRendezvousState::Restarting { restart_id };
|
||||
entry.tx.send_replace(entry.state.clone());
|
||||
true
|
||||
}
|
||||
|
||||
/// Publishes that the replacement structured session can accept continuation.
|
||||
pub fn ready(&self, key: RuntimeAgentKey, restart_id: &str) {
|
||||
self.transition(key, restart_id, |restart_id| {
|
||||
RestartRendezvousState::Ready { restart_id }
|
||||
});
|
||||
}
|
||||
|
||||
fn resume_taken(&self, key: RuntimeAgentKey, restart_id: &str) {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(entry) = entries.get_mut(&key) else {
|
||||
return;
|
||||
};
|
||||
if !matches!(
|
||||
&entry.state,
|
||||
RestartRendezvousState::Ready { restart_id: current } if current == restart_id
|
||||
) {
|
||||
return;
|
||||
}
|
||||
entry.state = RestartRendezvousState::Idle;
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
entry.tx.send_replace(entry.state.clone());
|
||||
}
|
||||
|
||||
/// Publishes an explicit restart failure to the awaiting logical rendezvous.
|
||||
pub fn fail(&self, key: RuntimeAgentKey, restart_id: &str, message: String) {
|
||||
self.transition(key, restart_id, |restart_id| {
|
||||
RestartRendezvousState::Failed {
|
||||
restart_id,
|
||||
message,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Cancels the restart-aware continuation for a target being explicitly stopped.
|
||||
/// Returns whether a restart handoff was active.
|
||||
pub fn cancel(&self, key: RuntimeAgentKey) -> bool {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(entry) = entries.get_mut(&key) else {
|
||||
return false;
|
||||
};
|
||||
let restart_id = match &entry.state {
|
||||
RestartRendezvousState::Restarting { restart_id }
|
||||
| RestartRendezvousState::Ready { restart_id } => restart_id.clone(),
|
||||
_ => return false,
|
||||
};
|
||||
entry.state = RestartRendezvousState::Cancelled { restart_id };
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
entry.tx.send_replace(entry.state.clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn transition(
|
||||
&self,
|
||||
key: RuntimeAgentKey,
|
||||
restart_id: &str,
|
||||
next: impl FnOnce(String) -> RestartRendezvousState,
|
||||
) {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(entry) = entries.get_mut(&key) else {
|
||||
return;
|
||||
};
|
||||
let matches = match &entry.state {
|
||||
RestartRendezvousState::Restarting {
|
||||
restart_id: current,
|
||||
}
|
||||
| RestartRendezvousState::Ready {
|
||||
restart_id: current,
|
||||
} => current == restart_id,
|
||||
_ => false,
|
||||
};
|
||||
if !matches {
|
||||
return;
|
||||
}
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
entry.state = next(restart_id.to_owned());
|
||||
entry.tx.send_replace(entry.state.clone());
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
/// Reports whether the owning rendezvous was cancelled during this restart.
|
||||
pub fn is_cancelled(&self, key: RuntimeAgentKey, restart_id: &str) -> bool {
|
||||
self.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&key)
|
||||
.is_some_and(|entry| {
|
||||
matches!(
|
||||
&entry.state,
|
||||
RestartRendezvousState::Cancelled { restart_id: current }
|
||||
if current == restart_id
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Waits until the active ask has accepted the replacement session handoff.
|
||||
pub async fn wait_for_handoff(
|
||||
&self,
|
||||
key: RuntimeAgentKey,
|
||||
restart_id: &str,
|
||||
) -> Result<(), String> {
|
||||
let mut rx = {
|
||||
let entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
entries
|
||||
.get(&key)
|
||||
.ok_or_else(|| "restart rendezvous disappeared".to_owned())?
|
||||
.tx
|
||||
.subscribe()
|
||||
};
|
||||
loop {
|
||||
let state = rx.borrow_and_update().clone();
|
||||
match state {
|
||||
RestartRendezvousState::Idle => return Ok(()),
|
||||
RestartRendezvousState::Cancelled {
|
||||
restart_id: current,
|
||||
} if current == restart_id => {
|
||||
return Err("restart rendezvous was cancelled".to_owned());
|
||||
}
|
||||
RestartRendezvousState::Failed {
|
||||
restart_id: current,
|
||||
message,
|
||||
} if current == restart_id => return Err(message),
|
||||
_ => {}
|
||||
}
|
||||
rx.changed()
|
||||
.await
|
||||
.map_err(|_| "restart rendezvous channel closed".to_owned())?;
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases terminal coordination state after the detached scheduler observed it.
|
||||
pub fn finish(&self, key: RuntimeAgentKey, restart_id: &str) {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let remove = entries.get(&key).is_some_and(|entry| {
|
||||
entry.active_waiters == 0
|
||||
&& matches!(
|
||||
&entry.state,
|
||||
RestartRendezvousState::Cancelled { restart_id: current }
|
||||
| RestartRendezvousState::Failed { restart_id: current, .. }
|
||||
if current == restart_id
|
||||
)
|
||||
});
|
||||
if remove {
|
||||
entries.remove(&key);
|
||||
}
|
||||
}
|
||||
|
||||
/// A restarting target is control-plane activity on every watchdog probe, even
|
||||
/// when provider/transcript activity is temporarily impossible during teardown.
|
||||
fn activity_token(&self, key: RuntimeAgentKey) -> Option<u64> {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let entry = entries.get_mut(&key)?;
|
||||
if matches!(
|
||||
entry.state,
|
||||
RestartRendezvousState::Restarting { .. } | RestartRendezvousState::Ready { .. }
|
||||
) {
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
}
|
||||
Some(entry.activity)
|
||||
}
|
||||
}
|
||||
|
||||
struct RestartRendezvousRegistration {
|
||||
coordinator: Arc<RestartRendezvousCoordinator>,
|
||||
key: RuntimeAgentKey,
|
||||
rx: watch::Receiver<RestartRendezvousState>,
|
||||
}
|
||||
|
||||
impl RestartRendezvousRegistration {
|
||||
fn state(&mut self) -> RestartRendezvousState {
|
||||
self.rx.borrow_and_update().clone()
|
||||
}
|
||||
|
||||
async fn changed(&mut self) -> Result<(), watch::error::RecvError> {
|
||||
self.rx.changed().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RestartRendezvousRegistration {
|
||||
fn drop(&mut self) {
|
||||
let mut entries = self
|
||||
.coordinator
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let Some(entry) = entries.get_mut(&self.key) else {
|
||||
return;
|
||||
};
|
||||
entry.active_waiters = entry.active_waiters.saturating_sub(1);
|
||||
if entry.active_waiters == 0 {
|
||||
let restart_id = match &entry.state {
|
||||
RestartRendezvousState::Restarting { restart_id }
|
||||
| RestartRendezvousState::Ready { restart_id } => Some(restart_id.clone()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(restart_id) = restart_id {
|
||||
entry.state = RestartRendezvousState::Cancelled { restart_id };
|
||||
entry.activity = entry.activity.saturating_add(1);
|
||||
entry.tx.send_replace(entry.state.clone());
|
||||
} else if matches!(
|
||||
entry.state,
|
||||
RestartRendezvousState::Idle | RestartRendezvousState::Failed { .. }
|
||||
) {
|
||||
entries.remove(&self.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Default terminal geometry for an orchestrator-launched agent cell. The UI
|
||||
/// resizes the PTY to the real cell size on attach; these are sane starting rows
|
||||
/// /cols so the spawn never fails on a zero-sized terminal.
|
||||
@ -706,6 +1030,8 @@ pub struct OrchestratorService {
|
||||
spawn_background_command: Option<Arc<SpawnBackgroundCommand>>,
|
||||
/// Detached self-restart coordinator, wired at the composition root.
|
||||
self_restart: Option<Arc<dyn AgentSelfRestartScheduler>>,
|
||||
/// Stable control-plane handoff between an active ask and a detached restart.
|
||||
restart_rendezvous: Arc<RestartRendezvousCoordinator>,
|
||||
}
|
||||
|
||||
/// Bundle des quatre use cases C7 sous [`domain::fileguard::FileGuard`], injectés
|
||||
@ -806,6 +1132,7 @@ impl OrchestratorService {
|
||||
session_limits: None,
|
||||
spawn_background_command: None,
|
||||
self_restart: None,
|
||||
restart_rendezvous: Arc::new(RestartRendezvousCoordinator::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -837,6 +1164,16 @@ impl OrchestratorService {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
/// Shares the runtime-keyed restart handoff with the detached restart adapter.
|
||||
pub fn with_restart_rendezvous(
|
||||
mut self,
|
||||
coordinator: Arc<RestartRendezvousCoordinator>,
|
||||
) -> Self {
|
||||
self.restart_rendezvous = coordinator;
|
||||
self
|
||||
}
|
||||
|
||||
/// Branche le store des tâches de fond pour tracer les rendez-vous inter-agents
|
||||
/// comme [`BackgroundTaskKind::HeadlessRendezvous`] (B6).
|
||||
///
|
||||
@ -2393,7 +2730,7 @@ impl OrchestratorService {
|
||||
conversation_id,
|
||||
requester,
|
||||
task,
|
||||
session.as_ref(),
|
||||
session,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@ -2419,7 +2756,7 @@ impl OrchestratorService {
|
||||
conversation_id: domain::conversation::ConversationId,
|
||||
requester: Option<AgentId>,
|
||||
task: String,
|
||||
session: &dyn domain::ports::AgentSession,
|
||||
session: Arc<dyn domain::ports::AgentSession>,
|
||||
) -> Result<OrchestratorOutcome, AppError> {
|
||||
let (input, mailbox) = match (&self.input, &self.mailbox) {
|
||||
(Some(i), Some(m)) => (i, m),
|
||||
@ -2524,57 +2861,131 @@ impl OrchestratorService {
|
||||
ticket: ticket_id,
|
||||
});
|
||||
let local_activity = Arc::new(AtomicU64::new(0));
|
||||
let drain = drain_with_readiness_announcements_and_activity_outcome(
|
||||
session,
|
||||
&task,
|
||||
None,
|
||||
input.as_ref(),
|
||||
agent_key,
|
||||
announcement_publisher,
|
||||
Some(Arc::clone(&local_activity)),
|
||||
let mut restart = self.restart_rendezvous.register(agent_key);
|
||||
let structured = Arc::clone(
|
||||
self.structured
|
||||
.as_ref()
|
||||
.expect("ask_structured requires the structured registry"),
|
||||
);
|
||||
let wait_activity = Arc::clone(&local_activity);
|
||||
|
||||
// L'attente du rendez-vous structured, rendue comme `Result<String, AppError>`
|
||||
// pour être enveloppée par le watchdog. Un flux clos sans `Final` correspond
|
||||
// au no-reply du chemin headless : la cible a rendu la main sans réponse
|
||||
// exploitable, on expose donc l'erreur métier retryable plutôt qu'un PROCESS.
|
||||
// The concrete provider session may disappear under us when the target calls
|
||||
// `idea_restart_self`. The logical rendezvous remains registered by
|
||||
// RuntimeAgentKey; on the restart control signal we drop only the old drain,
|
||||
// wait for the replacement session, then capture its continuation `Final`.
|
||||
let wait = async {
|
||||
match drain.await {
|
||||
Ok(TurnOutcome::Completed(content)) => Ok(content),
|
||||
Ok(TurnOutcome::RateLimited { resets_at_ms }) => {
|
||||
let conversation_id = session.conversation_id();
|
||||
let (conversation_id, resets_at_ms) = match (conversation_id, resets_at_ms) {
|
||||
(Some(conversation_id), resets_at_ms) => {
|
||||
(Some(conversation_id), resets_at_ms)
|
||||
let mut current_session = session;
|
||||
let mut prompt = task.clone();
|
||||
loop {
|
||||
let turn = {
|
||||
let drain = drain_with_readiness_announcements_and_activity_outcome(
|
||||
current_session.as_ref(),
|
||||
&prompt,
|
||||
None,
|
||||
input.as_ref(),
|
||||
agent_key,
|
||||
announcement_publisher.clone(),
|
||||
Some(Arc::clone(&wait_activity)),
|
||||
);
|
||||
tokio::pin!(drain);
|
||||
tokio::select! {
|
||||
biased;
|
||||
changed = restart.changed() => {
|
||||
changed.map_err(|_| AppError::Process(
|
||||
format!("restart coordination closed for target {target}")
|
||||
))?;
|
||||
None
|
||||
}
|
||||
(None, None) => (None, None),
|
||||
// Reset connu mais conversation moteur absente : ne pas armer
|
||||
// une reprise non reprenable ; surfacer le fallback humain.
|
||||
(None, Some(_)) => (None, None),
|
||||
};
|
||||
if let (Some(service), Some(structured)) =
|
||||
(&self.session_limits, &self.structured)
|
||||
{
|
||||
if let Some(node_id) =
|
||||
structured.node_for_agent_in_project(project.id, &agent_id)
|
||||
{
|
||||
service.on_rate_limited(
|
||||
project.id,
|
||||
agent_id,
|
||||
node_id,
|
||||
conversation_id,
|
||||
resets_at_ms,
|
||||
);
|
||||
outcome = &mut drain => Some(outcome),
|
||||
}
|
||||
};
|
||||
|
||||
let control_active = !matches!(restart.state(), RestartRendezvousState::Idle);
|
||||
if let Some(outcome) = turn.filter(|_| !control_active) {
|
||||
match outcome {
|
||||
Ok(TurnOutcome::Completed(content)) => return Ok(content),
|
||||
Ok(TurnOutcome::RateLimited { resets_at_ms }) => {
|
||||
let conversation_id = current_session.conversation_id();
|
||||
let (conversation_id, resets_at_ms) =
|
||||
match (conversation_id, resets_at_ms) {
|
||||
(Some(conversation_id), resets_at_ms) => {
|
||||
(Some(conversation_id), resets_at_ms)
|
||||
}
|
||||
(None, None) => (None, None),
|
||||
// Reset connu mais conversation moteur absente : ne pas armer
|
||||
// une reprise non reprenable ; surfacer le fallback humain.
|
||||
(None, Some(_)) => (None, None),
|
||||
};
|
||||
if let (Some(service), Some(structured)) =
|
||||
(&self.session_limits, &self.structured)
|
||||
{
|
||||
if let Some(node_id) =
|
||||
structured.node_for_agent_in_project(project.id, &agent_id)
|
||||
{
|
||||
service.on_rate_limited(
|
||||
project.id,
|
||||
agent_id,
|
||||
node_id,
|
||||
conversation_id,
|
||||
resets_at_ms,
|
||||
);
|
||||
}
|
||||
}
|
||||
return Err(AppError::Process(
|
||||
"le tour s'est clos en limite de session, sans contenu Final"
|
||||
.to_owned(),
|
||||
));
|
||||
}
|
||||
Err(err) if structured_no_reply_error(&err) => {
|
||||
return Err(AppError::TargetReturnedNoReply(target.to_owned()));
|
||||
}
|
||||
Err(err) => return Err(AppError::from(err)),
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
match restart.state() {
|
||||
RestartRendezvousState::Restarting { .. } => {
|
||||
restart.changed().await.map_err(|_| {
|
||||
AppError::Process(format!(
|
||||
"restart coordination closed for target {target}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
RestartRendezvousState::Ready { restart_id } => {
|
||||
current_session = structured
|
||||
.session_for_agent_in_project(project.id, &agent_id)
|
||||
.ok_or_else(|| {
|
||||
AppError::Process(format!(
|
||||
"restart completed without a live structured session for target {target}"
|
||||
))
|
||||
})?;
|
||||
self.bind_conversation_session(conversation_id, current_session.id());
|
||||
self.restart_rendezvous.resume_taken(agent_key, &restart_id);
|
||||
let _ = restart.state();
|
||||
prompt = SELF_RESTART_CONTINUATION_PROMPT.to_owned();
|
||||
wait_activity.fetch_add(1, AtomicOrdering::Relaxed);
|
||||
break;
|
||||
}
|
||||
RestartRendezvousState::Failed { message, .. } => {
|
||||
return Err(AppError::Process(format!(
|
||||
"restart failed for target {target}: {message}"
|
||||
)));
|
||||
}
|
||||
RestartRendezvousState::Cancelled { .. } => {
|
||||
return Err(AppError::Process(format!(
|
||||
"restart cancelled for target {target}"
|
||||
)));
|
||||
}
|
||||
RestartRendezvousState::Idle => {
|
||||
restart.changed().await.map_err(|_| {
|
||||
AppError::Process(format!(
|
||||
"restart coordination closed for target {target}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Err(AppError::Process(
|
||||
"le tour s'est clos en limite de session, sans contenu Final".to_owned(),
|
||||
))
|
||||
}
|
||||
Err(err) if structured_no_reply_error(&err) => {
|
||||
Err(AppError::TargetReturnedNoReply(target.to_owned()))
|
||||
}
|
||||
Err(err) => Err(AppError::from(err)),
|
||||
}
|
||||
};
|
||||
|
||||
@ -2590,6 +3001,7 @@ impl OrchestratorService {
|
||||
started,
|
||||
local: local_activity,
|
||||
},
|
||||
Some(agent_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@ -3278,6 +3690,10 @@ impl OrchestratorService {
|
||||
}
|
||||
}
|
||||
|
||||
let restart_cancelled = self
|
||||
.restart_rendezvous
|
||||
.cancel(runtime_key(project, agent_id));
|
||||
|
||||
let structured = self
|
||||
.structured
|
||||
.clone()
|
||||
@ -3317,6 +3733,12 @@ impl OrchestratorService {
|
||||
.await?
|
||||
.is_empty()
|
||||
{
|
||||
if restart_cancelled {
|
||||
return Ok(OrchestratorOutcome {
|
||||
detail: format!("cancelled restarting work for agent {label}"),
|
||||
reply: None,
|
||||
});
|
||||
}
|
||||
return Err(AppError::NotFound(format!(
|
||||
"running session for agent {label}"
|
||||
)));
|
||||
@ -3547,6 +3969,7 @@ impl OrchestratorService {
|
||||
agent_id: AgentId,
|
||||
target: &str,
|
||||
activity: AskWatchdogActivity,
|
||||
restart_key: Option<RuntimeAgentKey>,
|
||||
) -> WatchdogOutcome<R> {
|
||||
let AskWatchdogActivity {
|
||||
started,
|
||||
@ -3559,11 +3982,13 @@ impl OrchestratorService {
|
||||
#[derive(Default)]
|
||||
struct CompositeActivity {
|
||||
local: u64,
|
||||
restart: u64,
|
||||
transcript: Option<u64>,
|
||||
token: u64,
|
||||
}
|
||||
|
||||
let probe = self.ask_liveness_probe.clone();
|
||||
let restart_rendezvous = Arc::clone(&self.restart_rendezvous);
|
||||
let root = project_root.clone();
|
||||
let composite = Arc::new(StdMutex::new(CompositeActivity::default()));
|
||||
let probe_fn = move || {
|
||||
@ -3571,20 +3996,26 @@ impl OrchestratorService {
|
||||
let root = root.clone();
|
||||
let local_activity = Arc::clone(&local_activity);
|
||||
let composite = Arc::clone(&composite);
|
||||
let restart_rendezvous = Arc::clone(&restart_rendezvous);
|
||||
async move {
|
||||
let transcript = match &probe {
|
||||
Some(p) => p(root, agent_id).await,
|
||||
None => None,
|
||||
};
|
||||
let local = local_activity.load(AtomicOrdering::Relaxed);
|
||||
let restart = restart_key
|
||||
.and_then(|key| restart_rendezvous.activity_token(key))
|
||||
.unwrap_or(0);
|
||||
let mut state = composite.lock().unwrap();
|
||||
let local_delta = local.saturating_sub(state.local);
|
||||
let restart_delta = restart.saturating_sub(state.restart);
|
||||
let transcript_delta = match (state.transcript, transcript) {
|
||||
(Some(previous), Some(current)) => current.saturating_sub(previous),
|
||||
(None, Some(_)) => 1,
|
||||
_ => 0,
|
||||
};
|
||||
state.local = state.local.max(local);
|
||||
state.restart = state.restart.max(restart);
|
||||
if let Some(current) = transcript {
|
||||
state.transcript =
|
||||
Some(state.transcript.map_or(current, |old| old.max(current)));
|
||||
@ -3592,6 +4023,7 @@ impl OrchestratorService {
|
||||
state.token = state
|
||||
.token
|
||||
.saturating_add(local_delta)
|
||||
.saturating_add(restart_delta)
|
||||
.saturating_add(transcript_delta);
|
||||
Some(state.token)
|
||||
}
|
||||
@ -4057,4 +4489,34 @@ mod tests {
|
||||
"pas de cancel_head après disarm"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_control_activity_rearms_inactivity_watchdog() {
|
||||
let coordinator = Arc::new(RestartRendezvousCoordinator::new());
|
||||
let key = rkey(9);
|
||||
let _registration = coordinator.register(key);
|
||||
assert!(coordinator.begin(key, "restart-watchdog".to_owned()));
|
||||
let probe = Arc::clone(&coordinator);
|
||||
|
||||
let outcome = run_inactivity_watchdog(
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(45)).await;
|
||||
"continued"
|
||||
},
|
||||
Duration::from_millis(10),
|
||||
Duration::from_millis(100),
|
||||
Instant::now(),
|
||||
true,
|
||||
move || {
|
||||
let probe = Arc::clone(&probe);
|
||||
async move { probe.activity_token(key) }
|
||||
},
|
||||
|_| {},
|
||||
|_| {},
|
||||
|_| {},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(outcome, WatchdogOutcome::Resolved("continued")));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user