fix(orchestrator,agent): rendezvous cross-project réarmé par jeton d'activité live model-agnostique — #287 (le watchdog d'inactivité du rendez-vous expirait à 600s sur un tour structured actif car seul le transcript Claude servait de signe de vie; nouveau drain_with_readiness_announcements_and_activity_outcome exposant un jeton monotone AtomicU64 incrémenté par chaque ReplyEvent non terminal du tap send_with_tap, passé au watchdog via AskWatchdogActivity pour réarmer la fenêtre quelle que soit l'adapter; la sonde transcript devient fallback externe; QA verte: orchestrator_service 81/81, rendezvous 5/5, mcp_server 40/40, backend 87+7 ignored, cargo fmt --check; scripts/ hors périmètre)
This commit is contained in:
@ -23,7 +23,8 @@ pub use session_limit::{AgentResumer, SessionLimitService, RESUME_PROMPT};
|
||||
pub use structured::{
|
||||
drain_reply_stream_with_readiness, drain_with_readiness,
|
||||
drain_with_readiness_and_announcements, drain_with_readiness_and_announcements_outcome,
|
||||
drain_with_readiness_outcome, send_blocking, AnnouncementPublisher, TurnOutcome,
|
||||
drain_with_readiness_announcements_and_activity_outcome, drain_with_readiness_outcome,
|
||||
send_blocking, AnnouncementPublisher, TurnOutcome,
|
||||
};
|
||||
|
||||
pub use catalogue::{
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
//! DRY : un seul chemin de lecture (le flux). `send_blocking` n'est qu'un *consom-
|
||||
//! mateur* du même flux que la cellule chat utilise pour le rendu incrémental.
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
@ -203,15 +204,49 @@ pub async fn drain_with_readiness_and_announcements_outcome(
|
||||
agent: RuntimeAgentKey,
|
||||
announcements: Option<AnnouncementPublisher>,
|
||||
) -> Result<TurnOutcome, AgentSessionError> {
|
||||
let Some(publisher) = announcements else {
|
||||
if announcements.is_none() {
|
||||
return drain_with_readiness_outcome(session, prompt, timeout, mediator, agent).await;
|
||||
};
|
||||
}
|
||||
drain_with_readiness_announcements_and_activity_outcome(
|
||||
session,
|
||||
prompt,
|
||||
timeout,
|
||||
mediator,
|
||||
agent,
|
||||
announcements,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Variante du drain d'annonces qui expose aussi un jeton monotone d'activité live.
|
||||
///
|
||||
/// Le jeton avance pour chaque [`ReplyEvent`] non terminal reçu via
|
||||
/// [`AgentSession::send_with_tap`], donc pendant que `send_with_tap` travaille encore
|
||||
/// et avant que son flux batch final soit disponible. Il permet au watchdog du
|
||||
/// rendez-vous de distinguer un tour structured actif d'un vrai silence, quel que
|
||||
/// soit le provider. `Final` reste exclusivement traité par le drain normal.
|
||||
pub async fn drain_with_readiness_announcements_and_activity_outcome(
|
||||
session: &dyn AgentSession,
|
||||
prompt: &str,
|
||||
timeout: Option<Duration>,
|
||||
mediator: &dyn InputMediator,
|
||||
agent: RuntimeAgentKey,
|
||||
announcements: Option<AnnouncementPublisher>,
|
||||
activity: Option<Arc<AtomicU64>>,
|
||||
) -> Result<TurnOutcome, AgentSessionError> {
|
||||
let (tap_tx, tap_rx) = std::sync::mpsc::channel();
|
||||
let live_publisher = publisher.clone();
|
||||
let live_publisher = announcements.clone();
|
||||
let live_pump = std::thread::spawn(move || {
|
||||
for event in tap_rx {
|
||||
live_publisher.publish_event(&event);
|
||||
if !matches!(event, ReplyEvent::Final { .. }) {
|
||||
if let Some(activity) = &activity {
|
||||
activity.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
if let Some(publisher) = &live_publisher {
|
||||
publisher.publish_event(&event);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@ -230,7 +265,9 @@ pub async fn drain_with_readiness_and_announcements_outcome(
|
||||
if !matches!(event, ReplyEvent::Final { .. }) {
|
||||
mediator.mark_alive(agent);
|
||||
}
|
||||
publisher.publish_event(event);
|
||||
if let Some(publisher) = &announcements {
|
||||
publisher.publish_event(event);
|
||||
}
|
||||
},
|
||||
|signal| {
|
||||
if signal == ReadinessSignal::TurnEnded {
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@ -40,10 +41,10 @@ use crate::conversation::RecordTurn;
|
||||
use crate::memory::HarvestMemoryFromTurn;
|
||||
|
||||
use crate::agent::{
|
||||
drain_with_readiness_and_announcements_outcome, AnnouncementPublisher, CreateAgentFromScratch,
|
||||
CreateAgentInput, LaunchAgent, LaunchAgentInput, ListAgents, ListAgentsInput,
|
||||
ManagedTempExposureRequest, McpRuntime, ReattachDecision, SessionLimitService, TurnOutcome,
|
||||
UpdateAgentContext, UpdateAgentContextInput, CODEX_SUBMIT_DELAY_MS,
|
||||
drain_with_readiness_announcements_and_activity_outcome, AnnouncementPublisher,
|
||||
CreateAgentFromScratch, CreateAgentInput, LaunchAgent, LaunchAgentInput, ListAgents,
|
||||
ListAgentsInput, ManagedTempExposureRequest, McpRuntime, ReattachDecision, SessionLimitService,
|
||||
TurnOutcome, UpdateAgentContext, UpdateAgentContextInput, CODEX_SUBMIT_DELAY_MS,
|
||||
};
|
||||
use crate::background::{SpawnBackgroundCommand, SpawnBackgroundCommandInput};
|
||||
use crate::error::AppError;
|
||||
@ -209,12 +210,11 @@ const ASK_AGENT_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
/// `None` ⇒ vivacité non observable (cible/transcript introuvable) ⇒ traité comme
|
||||
/// « pas de progrès ».
|
||||
///
|
||||
/// **Pourquoi un port et pas le flux d'événements** : le flux de
|
||||
/// [`domain::ports::AgentSession::send`] est **batché en fin de tour** (les `ReplyEvent`
|
||||
/// arrivent tous à la fin, pas en continu), donc inutilisable comme signe de vie *pendant*
|
||||
/// un long tour. La seule preuve de vie disponible est cette sonde externe (transcript).
|
||||
/// Closure abstraite (frontière hexagonale) construite à la composition root — la seule
|
||||
/// couche qui sait dériver le run-dir transcript d'un `AgentId` ; l'application reste pure.
|
||||
/// Sur le chemin structured, cette sonde externe complète le tap live
|
||||
/// [`domain::ports::AgentSession::send_with_tap`] : elle reste utile lorsque l'adapter
|
||||
/// ne produit aucun événement intermédiaire. Closure abstraite (frontière hexagonale)
|
||||
/// construite à la composition root — la seule couche qui sait dériver le run-dir
|
||||
/// transcript d'un `AgentId` ; l'application reste pure.
|
||||
pub type AskLivenessProbe = Arc<
|
||||
dyn Fn(
|
||||
domain::project::ProjectPath,
|
||||
@ -224,6 +224,11 @@ pub type AskLivenessProbe = Arc<
|
||||
+ Sync,
|
||||
>;
|
||||
|
||||
struct AskWatchdogActivity {
|
||||
started: Instant,
|
||||
local: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
/// Plafond **absolu** par défaut du rendez-vous délégué : la fenêtre d'inactivité
|
||||
/// réarmée ne parque jamais un `ask` au-delà, même contre une cible perpétuellement
|
||||
/// active. Généreux (4 h) pour qu'un tour unique lourd mais réel finisse bien avant.
|
||||
@ -2470,13 +2475,15 @@ impl OrchestratorService {
|
||||
target: agent_id,
|
||||
ticket: ticket_id,
|
||||
});
|
||||
let drain = drain_with_readiness_and_announcements_outcome(
|
||||
let local_activity = Arc::new(AtomicU64::new(0));
|
||||
let drain = drain_with_readiness_announcements_and_activity_outcome(
|
||||
session,
|
||||
&task,
|
||||
None,
|
||||
input.as_ref(),
|
||||
agent_key,
|
||||
announcement_publisher,
|
||||
Some(Arc::clone(&local_activity)),
|
||||
);
|
||||
|
||||
// L'attente du rendez-vous structured, rendue comme `Result<String, AppError>`
|
||||
@ -2525,7 +2532,17 @@ impl OrchestratorService {
|
||||
|
||||
// Borne par la fenêtre d'inactivité (réarmée sur signe de vie) sous plafond absolu.
|
||||
let result = match self
|
||||
.run_ask_with_watchdog(wait, turn_timeout, &project.root, agent_id, target, started)
|
||||
.run_ask_with_watchdog(
|
||||
wait,
|
||||
turn_timeout,
|
||||
&project.root,
|
||||
agent_id,
|
||||
target,
|
||||
AskWatchdogActivity {
|
||||
started,
|
||||
local: local_activity,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
WatchdogOutcome::Resolved(Ok(content)) => {
|
||||
@ -3481,19 +3498,54 @@ impl OrchestratorService {
|
||||
project_root: &domain::project::ProjectPath,
|
||||
agent_id: AgentId,
|
||||
target: &str,
|
||||
started: Instant,
|
||||
activity: AskWatchdogActivity,
|
||||
) -> WatchdogOutcome<R> {
|
||||
let has_probe = self.ask_liveness_probe.is_some();
|
||||
let AskWatchdogActivity {
|
||||
started,
|
||||
local: local_activity,
|
||||
} = activity;
|
||||
// Le tap structured est toujours observable ; le transcript reste un fallback
|
||||
// utile pour les providers qui ne produisent aucun événement live pendant un
|
||||
// long tour. Les deux sources sont fusionnées en un jeton monotone cumulatif :
|
||||
// la disparition/régression de l'une ne peut jamais masquer le progrès de l'autre.
|
||||
#[derive(Default)]
|
||||
struct CompositeActivity {
|
||||
local: u64,
|
||||
transcript: Option<u64>,
|
||||
token: u64,
|
||||
}
|
||||
|
||||
let probe = self.ask_liveness_probe.clone();
|
||||
let root = project_root.clone();
|
||||
let composite = Arc::new(StdMutex::new(CompositeActivity::default()));
|
||||
let probe_fn = move || {
|
||||
let probe = probe.clone();
|
||||
let root = root.clone();
|
||||
let local_activity = Arc::clone(&local_activity);
|
||||
let composite = Arc::clone(&composite);
|
||||
async move {
|
||||
match &probe {
|
||||
let transcript = match &probe {
|
||||
Some(p) => p(root, agent_id).await,
|
||||
None => None,
|
||||
};
|
||||
let local = local_activity.load(AtomicOrdering::Relaxed);
|
||||
let mut state = composite.lock().unwrap();
|
||||
let local_delta = local.saturating_sub(state.local);
|
||||
let transcript_delta = match (state.transcript, transcript) {
|
||||
(Some(previous), Some(current)) => current.saturating_sub(previous),
|
||||
(None, Some(_)) => 1,
|
||||
_ => 0,
|
||||
};
|
||||
state.local = state.local.max(local);
|
||||
if let Some(current) = transcript {
|
||||
state.transcript =
|
||||
Some(state.transcript.map_or(current, |old| old.max(current)));
|
||||
}
|
||||
state.token = state
|
||||
.token
|
||||
.saturating_add(local_delta)
|
||||
.saturating_add(transcript_delta);
|
||||
Some(state.token)
|
||||
}
|
||||
};
|
||||
let ext_target = target.to_owned();
|
||||
@ -3506,7 +3558,7 @@ impl OrchestratorService {
|
||||
window,
|
||||
self.ask_ceiling,
|
||||
started,
|
||||
has_probe,
|
||||
true,
|
||||
probe_fn,
|
||||
move |elapsed| {
|
||||
crate::diag!(
|
||||
|
||||
Reference in New Issue
Block a user