fix(backend,orchestrator,chat): tour humain agent_send survit au self-restart — #290 (le tour humain direct agent_send/ws chat.send s'enregistre au RestartRendezvousCoordinator via StructuredHumanTurnGuard: Final/Error de l'ancien provider devient transition de contrôle et non le Final logique; transition_after_provider_terminal_async attend Ready puis accept_ready acquitte le handoff et libère le scheduler, Resume => drain de la session de remplacement avec SELF_RESTART_CONTINUATION_PROMPT, seul le terminal du remplacement est transmis au chat; Failed/Cancelled surfacés explicitement avec libération Busy/FIFO; send_with_tap en erreur pendant restart actif traité comme teardown attendu (stream vide puis attente du remplacement, plus de Final prématuré ni chat bloqué); StopLiveAgent câblé with_restart_rendezvous: stop utilisateur annule le rendez-vous restart; RestartRendezvousRegistration rendu public (state/changed/accept_ready); QA verte: backend self_restart_contract_tests 3/3 dont 2 nouveaux (old terminal ignoré + handoff accepté, failure explicite + busy libéré), application orchestrator_service restart 8/8 dont registration directe du rendez-vous, workstate_actions 9/9, app-tauri chat_bridge 31/31, web-server websocket_chat_send 3/3, cargo check --workspace, fmt --check, diff --check — reconfirmés par Git avant commit; réserves hors périmètre préexistantes: 7 échecs fixture PTY web-server + 2 échecs plugin application; carnet #290 + clôture store #289 inclus; scripts/ hors périmètre; aucun push)
This commit is contained in:
@ -2570,8 +2570,10 @@ pub async fn managed_temp_gc(
|
||||
/// structured registry, open the turn stream (`session.send`), then drain that
|
||||
/// blocking [`ReplyStream`] on a dedicated OS thread, mapping each
|
||||
/// [`ReplyEvent`](domain::ports::ReplyEvent) to a [`ReplyChunk`] and forwarding it
|
||||
/// through the [`ChatBridge`]. The turn ends deterministically at
|
||||
/// [`ReplyChunk::Final`]; the stream is bounded and closes right after.
|
||||
/// through the [`ChatBridge`]. The logical turn ends deterministically at the
|
||||
/// authoritative [`ReplyChunk::Final`]. If the agent self-restarts, the old
|
||||
/// provider's terminal event is a control transition: the pump waits for the
|
||||
/// replacement session and only forwards its terminal outcome.
|
||||
///
|
||||
/// The channel is (re-)registered each call, bumping the bridge **generation** so
|
||||
/// a previous attach's pump can no longer deliver (generation supersede): only the
|
||||
@ -2621,6 +2623,7 @@ pub async fn agent_send(
|
||||
backend::StructuredHumanTurnGuard::start(
|
||||
std::sync::Arc::clone(&state.turn_watch_input),
|
||||
std::sync::Arc::clone(&state.input_mailbox),
|
||||
std::sync::Arc::clone(&state.restart_rendezvous),
|
||||
*project_id,
|
||||
*agent_id,
|
||||
&prompt_for_model,
|
||||
@ -2673,7 +2676,20 @@ pub async fn agent_send(
|
||||
// still live, so we keep the attach and surface the error.
|
||||
let stream_result = session.send_with_tap(&prompt_for_model, tap_tx).await;
|
||||
let _ = tap_pump.join();
|
||||
let stream = stream_result.map_err(|e| ErrorDto::from(AppError::from(e)))?;
|
||||
let mut turn_guard = turn_guard;
|
||||
let stream = match stream_result {
|
||||
Ok(stream) => stream,
|
||||
Err(_error)
|
||||
if turn_guard
|
||||
.as_mut()
|
||||
.is_some_and(|guard| guard.restart_is_active()) =>
|
||||
{
|
||||
// Teardown of the old provider is expected once self-restart owns the
|
||||
// logical turn. The detached drain below waits for the replacement.
|
||||
Box::new(std::iter::empty()) as domain::ports::ReplyStream
|
||||
}
|
||||
Err(error) => return Err(ErrorDto::from(AppError::from(error))),
|
||||
};
|
||||
|
||||
// Drain the blocking reply iterator on a dedicated OS thread (the stream is a
|
||||
// synchronous `Iterator`, exactly like the PTY byte stream). It runs to the
|
||||
@ -2685,52 +2701,144 @@ pub async fn agent_send(
|
||||
// event then feeds the service (détecter→planifier). Without a conversation id the
|
||||
// service must not pretend an automatic resume is armed.
|
||||
let service = std::sync::Arc::clone(&state.session_limit_service);
|
||||
let structured = std::sync::Arc::clone(&state.structured_sessions);
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
let meta = session_meta;
|
||||
std::thread::spawn(move || {
|
||||
let mut terminal_visible = false;
|
||||
let mut settler = crate::chat::ProgressSettler::default();
|
||||
for event in stream {
|
||||
if !matches!(event, domain::ports::ReplyEvent::Final { .. }) {
|
||||
if let Some(guard) = &turn_guard {
|
||||
guard.mark_alive();
|
||||
}
|
||||
}
|
||||
// Non-terminal, sans contenu chat : un signal de limite alimente le service
|
||||
// (le badge UI vient du bus `AgentRateLimited`, pas du flux chat) puis on
|
||||
// continue à drainer comme pour un battement.
|
||||
if let domain::ports::ReplyEvent::RateLimited { resets_at_ms } = &event {
|
||||
if let Some((project_id, agent_id, node_id, conversation_id)) = &meta {
|
||||
let (conversation_id, resets_at_ms) = match (conversation_id, resets_at_ms) {
|
||||
(Some(conversation_id), resets_at_ms) => {
|
||||
(Some(conversation_id.clone()), *resets_at_ms)
|
||||
let mut current_stream = stream;
|
||||
'logical_turn: loop {
|
||||
let mut concrete_terminal = false;
|
||||
let mut resume = false;
|
||||
for event in current_stream {
|
||||
let provider_terminal = matches!(
|
||||
event,
|
||||
domain::ports::ReplyEvent::Final { .. }
|
||||
| domain::ports::ReplyEvent::Error { .. }
|
||||
);
|
||||
if provider_terminal {
|
||||
concrete_terminal = true;
|
||||
if let Some(guard) = turn_guard.as_mut() {
|
||||
match runtime.block_on(guard.transition_after_provider_terminal_async()) {
|
||||
backend::StructuredHumanTurnTransition::Complete => {}
|
||||
backend::StructuredHumanTurnTransition::Resume => {
|
||||
resume = true;
|
||||
break;
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Failed(message) => {
|
||||
terminal_visible = true;
|
||||
let _ = bridge.send_output_settled(
|
||||
&sid,
|
||||
&mut settler,
|
||||
ReplyChunk::Error { message },
|
||||
);
|
||||
break 'logical_turn;
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Cancelled => {
|
||||
terminal_visible = true;
|
||||
break 'logical_turn;
|
||||
}
|
||||
}
|
||||
(None, None) => (None, None),
|
||||
// Reset connu mais conversation moteur absente : ne pas armer
|
||||
// une reprise non reprenable ; surfacer le fallback humain.
|
||||
(None, Some(_)) => (None, None),
|
||||
};
|
||||
service.on_rate_limited(
|
||||
*project_id,
|
||||
*agent_id,
|
||||
*node_id,
|
||||
conversation_id,
|
||||
resets_at_ms,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if let Some(guard) = &turn_guard {
|
||||
guard.mark_alive();
|
||||
}
|
||||
}
|
||||
// Non-terminal, sans contenu chat : un signal de limite alimente le service
|
||||
// (le badge UI vient du bus `AgentRateLimited`, pas du flux chat) puis on
|
||||
// continue à drainer comme pour un battement.
|
||||
if let domain::ports::ReplyEvent::RateLimited { resets_at_ms } = &event {
|
||||
if let Some((project_id, agent_id, node_id, conversation_id)) = &meta {
|
||||
let (conversation_id, resets_at_ms) = match (conversation_id, resets_at_ms)
|
||||
{
|
||||
(Some(conversation_id), resets_at_ms) => {
|
||||
(Some(conversation_id.clone()), *resets_at_ms)
|
||||
}
|
||||
(None, None) => (None, None),
|
||||
// Reset connu mais conversation moteur absente : ne pas armer
|
||||
// une reprise non reprenable ; surfacer le fallback humain.
|
||||
(None, Some(_)) => (None, None),
|
||||
};
|
||||
service.on_rate_limited(
|
||||
*project_id,
|
||||
*agent_id,
|
||||
*node_id,
|
||||
conversation_id,
|
||||
resets_at_ms,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Heartbeats carry no chat content (readiness/heartbeat lot 1) ⇒ no wire
|
||||
// chunk; skip them while still draining so the turn runs to its `Final`.
|
||||
let Some(chunk) = crate::chat::chunk_from_event(event) else {
|
||||
continue;
|
||||
};
|
||||
if matches!(chunk, ReplyChunk::Final { .. } | ReplyChunk::Error { .. }) {
|
||||
terminal_visible = true;
|
||||
}
|
||||
// `send_output` always records into the conversation scrollback; the
|
||||
// boolean only reflects live delivery. If the view navigated away
|
||||
// (no channel at this generation) we keep draining so the turn still
|
||||
// completes and the scrollback stays whole for the next re-attach.
|
||||
let _ = bridge.send_output_settled(&sid, &mut settler, chunk);
|
||||
}
|
||||
|
||||
if !resume && !concrete_terminal {
|
||||
if let Some(guard) = turn_guard.as_mut() {
|
||||
match runtime.block_on(guard.transition_after_provider_terminal_async()) {
|
||||
backend::StructuredHumanTurnTransition::Complete => {}
|
||||
backend::StructuredHumanTurnTransition::Resume => resume = true,
|
||||
backend::StructuredHumanTurnTransition::Failed(message) => {
|
||||
terminal_visible = true;
|
||||
let _ = bridge.send_output_settled(
|
||||
&sid,
|
||||
&mut settler,
|
||||
ReplyChunk::Error { message },
|
||||
);
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Cancelled => {
|
||||
terminal_visible = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Heartbeats carry no chat content (readiness/heartbeat lot 1) ⇒ no wire
|
||||
// chunk; skip them while still draining so the turn runs to its `Final`.
|
||||
let Some(chunk) = crate::chat::chunk_from_event(event) else {
|
||||
continue;
|
||||
};
|
||||
if matches!(chunk, ReplyChunk::Final { .. } | ReplyChunk::Error { .. }) {
|
||||
terminal_visible = true;
|
||||
|
||||
if !resume {
|
||||
break;
|
||||
}
|
||||
let Some((project_id, agent_id, _, _)) = meta.as_ref() else {
|
||||
break;
|
||||
};
|
||||
let Some(replacement) = structured.session_for_agent_in_project(*project_id, agent_id)
|
||||
else {
|
||||
terminal_visible = true;
|
||||
let _ = bridge.send_output_settled(
|
||||
&sid,
|
||||
&mut settler,
|
||||
ReplyChunk::Error {
|
||||
message: "Le redémarrage n'a pas fourni de session de remplacement."
|
||||
.to_owned(),
|
||||
},
|
||||
);
|
||||
break;
|
||||
};
|
||||
match runtime.block_on(replacement.send(application::SELF_RESTART_CONTINUATION_PROMPT))
|
||||
{
|
||||
Ok(stream) => current_stream = stream,
|
||||
Err(error) => {
|
||||
terminal_visible = true;
|
||||
let _ = bridge.send_output_settled(
|
||||
&sid,
|
||||
&mut settler,
|
||||
ReplyChunk::Error {
|
||||
message: error.to_string(),
|
||||
},
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// `send_output` always records into the conversation scrollback; the
|
||||
// boolean only reflects live delivery. If the view navigated away
|
||||
// (no channel at this generation) we keep draining so the turn still
|
||||
// completes and the scrollback stays whole for the next re-attach.
|
||||
let _ = bridge.send_output_settled(&sid, &mut settler, chunk);
|
||||
}
|
||||
if !terminal_visible {
|
||||
let chunk = crate::chat::unfinished_turn_chunk();
|
||||
|
||||
@ -168,9 +168,9 @@ pub use orchestrator::{
|
||||
AgentRestartScheduled, AgentSelfRestartScheduler, AgentWakeService, AskLivenessProbe,
|
||||
ContextGuardUseCases, LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider,
|
||||
OrchestratorOutcome, OrchestratorService, ProposeContext, ReadContext, ReadMemory,
|
||||
RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState, WakeSessionProvider,
|
||||
WatchdogOutcome, WriteMemory, DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW,
|
||||
SELF_RESTART_CONTINUATION_PROMPT,
|
||||
RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousRegistration,
|
||||
RestartRendezvousState, WakeSessionProvider, WatchdogOutcome, WriteMemory,
|
||||
DEFAULT_RENDEZVOUS_CEILING, DEFAULT_RENDEZVOUS_WINDOW, SELF_RESTART_CONTINUATION_PROMPT,
|
||||
};
|
||||
pub use permission::{
|
||||
GetProjectPermissions, GetProjectPermissionsInput, GetProjectPermissionsOutput,
|
||||
|
||||
@ -21,7 +21,7 @@ pub use rendezvous::{
|
||||
pub use service::{
|
||||
AgentRestartScheduled, AgentSelfRestartScheduler, AskLivenessProbe, ContextGuardUseCases,
|
||||
LiveStateProvider, LiveStateReadProvider, McpRuntimeProvider, OrchestratorOutcome,
|
||||
OrchestratorService, RecordTurnProvider, RestartRendezvousCoordinator, RestartRendezvousState,
|
||||
SELF_RESTART_CONTINUATION_PROMPT,
|
||||
OrchestratorService, RecordTurnProvider, RestartRendezvousCoordinator,
|
||||
RestartRendezvousRegistration, RestartRendezvousState, SELF_RESTART_CONTINUATION_PROMPT,
|
||||
};
|
||||
pub use wake::{AgentWakeService, WakeSessionProvider};
|
||||
|
||||
@ -138,7 +138,9 @@ impl RestartRendezvousCoordinator {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
fn register(self: &Arc<Self>, key: RuntimeAgentKey) -> RestartRendezvousRegistration {
|
||||
/// Registers one logical turn that must survive replacement of its concrete
|
||||
/// provider session.
|
||||
pub fn register(self: &Arc<Self>, key: RuntimeAgentKey) -> RestartRendezvousRegistration {
|
||||
let mut entries = self
|
||||
.entries
|
||||
.lock()
|
||||
@ -359,20 +361,29 @@ impl RestartRendezvousCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
struct RestartRendezvousRegistration {
|
||||
/// Per-turn subscription to restart control-plane transitions.
|
||||
pub struct RestartRendezvousRegistration {
|
||||
coordinator: Arc<RestartRendezvousCoordinator>,
|
||||
key: RuntimeAgentKey,
|
||||
rx: watch::Receiver<RestartRendezvousState>,
|
||||
}
|
||||
|
||||
impl RestartRendezvousRegistration {
|
||||
fn state(&mut self) -> RestartRendezvousState {
|
||||
/// Returns the latest transition, marking it observed by this turn.
|
||||
pub fn state(&mut self) -> RestartRendezvousState {
|
||||
self.rx.borrow_and_update().clone()
|
||||
}
|
||||
|
||||
async fn changed(&mut self) -> Result<(), watch::error::RecvError> {
|
||||
/// Waits asynchronously for a restart control transition.
|
||||
pub async fn changed(&mut self) -> Result<(), watch::error::RecvError> {
|
||||
self.rx.changed().await
|
||||
}
|
||||
|
||||
/// Acknowledges ownership of the replacement session and releases the
|
||||
/// detached restart scheduler.
|
||||
pub fn accept_ready(&self, restart_id: &str) {
|
||||
self.coordinator.resume_taken(self.key, restart_id);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RestartRendezvousRegistration {
|
||||
|
||||
@ -149,6 +149,7 @@ pub struct StopLiveAgent {
|
||||
input: Option<Arc<dyn InputMediator>>,
|
||||
waits: Option<Arc<OrchestratorService>>,
|
||||
cascade: Option<Arc<dyn StopAgentExecutionCascade>>,
|
||||
restart_rendezvous: Option<Arc<crate::RestartRendezvousCoordinator>>,
|
||||
}
|
||||
|
||||
impl StopLiveAgent {
|
||||
@ -161,6 +162,7 @@ impl StopLiveAgent {
|
||||
input: None,
|
||||
waits: None,
|
||||
cascade: None,
|
||||
restart_rendezvous: None,
|
||||
}
|
||||
}
|
||||
|
||||
@ -187,6 +189,16 @@ impl StopLiveAgent {
|
||||
self
|
||||
}
|
||||
|
||||
/// Wires cancellation of a logical turn currently crossing a self-restart.
|
||||
#[must_use]
|
||||
pub fn with_restart_rendezvous(
|
||||
mut self,
|
||||
coordinator: Arc<crate::RestartRendezvousCoordinator>,
|
||||
) -> Self {
|
||||
self.restart_rendezvous = Some(coordinator);
|
||||
self
|
||||
}
|
||||
|
||||
/// Stops the agent's current work.
|
||||
///
|
||||
/// # Errors
|
||||
@ -198,6 +210,9 @@ impl StopLiveAgent {
|
||||
) -> Result<StopLiveAgentOutput, AppError> {
|
||||
self.stop_dependencies(&input.project, input.agent_id)
|
||||
.await?;
|
||||
if let Some(coordinator) = &self.restart_rendezvous {
|
||||
coordinator.cancel(RuntimeAgentKey::new(input.project.id, input.agent_id));
|
||||
}
|
||||
if let Some(mediator) = &self.input {
|
||||
mediator.preempt(RuntimeAgentKey::new(input.project.id, input.agent_id));
|
||||
}
|
||||
|
||||
@ -5716,6 +5716,60 @@ async fn same_project_stop_during_restart_releases_ask_and_leaves_no_replacement
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_human_turn_registration_holds_restart_until_replacement_handoff() {
|
||||
let coordinator = Arc::new(application::RestartRendezvousCoordinator::new());
|
||||
let key = rkey(1);
|
||||
let restart_id = "restart-human-turn".to_owned();
|
||||
let mut turn = coordinator.register(key);
|
||||
|
||||
assert!(
|
||||
coordinator.begin(key, restart_id.clone()),
|
||||
"a registered direct human turn owns self-restart continuation"
|
||||
);
|
||||
assert!(matches!(
|
||||
turn.state(),
|
||||
application::RestartRendezvousState::Restarting { .. }
|
||||
));
|
||||
|
||||
coordinator.ready(key, &restart_id);
|
||||
turn.changed().await.expect("ready transition remains live");
|
||||
assert_eq!(
|
||||
turn.state(),
|
||||
application::RestartRendezvousState::Ready {
|
||||
restart_id: restart_id.clone()
|
||||
}
|
||||
);
|
||||
|
||||
turn.accept_ready(&restart_id);
|
||||
coordinator
|
||||
.wait_for_handoff(key, &restart_id)
|
||||
.await
|
||||
.expect("scheduler is released only after the replacement is accepted");
|
||||
assert_eq!(turn.state(), application::RestartRendezvousState::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_human_turn_registration_surfaces_restart_failure() {
|
||||
let coordinator = Arc::new(application::RestartRendezvousCoordinator::new());
|
||||
let key = rkey(1);
|
||||
let restart_id = "restart-human-failure".to_owned();
|
||||
let mut turn = coordinator.register(key);
|
||||
|
||||
assert!(coordinator.begin(key, restart_id.clone()));
|
||||
coordinator.fail(key, &restart_id, "replacement launch failed".to_owned());
|
||||
turn.changed()
|
||||
.await
|
||||
.expect("failure transition remains live");
|
||||
assert_eq!(
|
||||
turn.state(),
|
||||
application::RestartRendezvousState::Failed {
|
||||
restart_id,
|
||||
message: "replacement launch failed".to_owned(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/// Régression #287 : la sonde transcript peut être stable (notamment inexistante pour
|
||||
/// un provider non-Claude), tandis que `send_with_tap` publie réellement l'activité du
|
||||
/// tour. Chaque événement non terminal doit réarmer la fenêtre jusqu'au `Final` tardif.
|
||||
|
||||
@ -1072,6 +1072,7 @@ fn self_restart_launch_input(
|
||||
#[cfg(test)]
|
||||
mod self_restart_contract_tests {
|
||||
use super::*;
|
||||
use domain::input::InputMediator;
|
||||
|
||||
#[test]
|
||||
fn relaunch_preserves_conversation_and_never_requests_fresh_start() {
|
||||
@ -1104,6 +1105,77 @@ mod self_restart_contract_tests {
|
||||
assert!(input.require_structured);
|
||||
assert!(!input.fresh_start);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn human_turn_ignores_old_terminal_and_accepts_replacement() {
|
||||
let mailbox = Arc::new(InMemoryMailbox::new());
|
||||
let input = Arc::new(MediatedInbox::new(
|
||||
Arc::clone(&mailbox),
|
||||
Arc::new(SystemMillisClock),
|
||||
));
|
||||
let coordinator = Arc::new(application::RestartRendezvousCoordinator::new());
|
||||
let project_id = ProjectId::from_uuid(Uuid::from_u128(290));
|
||||
let agent_id = AgentId::from_uuid(Uuid::from_u128(1));
|
||||
let key = domain::RuntimeAgentKey::new(project_id, agent_id);
|
||||
let mut turn = StructuredHumanTurnGuard::start(
|
||||
Arc::clone(&input) as Arc<dyn domain::input::InputMediator>,
|
||||
Arc::clone(&mailbox) as Arc<dyn domain::mailbox::AgentMailbox>,
|
||||
Arc::clone(&coordinator),
|
||||
project_id,
|
||||
agent_id,
|
||||
"restart me",
|
||||
);
|
||||
let restart_id = "restart-human-290".to_owned();
|
||||
|
||||
assert!(coordinator.begin(key, restart_id.clone()));
|
||||
coordinator.ready(key, &restart_id);
|
||||
assert_eq!(
|
||||
turn.transition_after_provider_terminal_async().await,
|
||||
StructuredHumanTurnTransition::Resume,
|
||||
"the old provider terminal is a control transition, not the human Final"
|
||||
);
|
||||
coordinator
|
||||
.wait_for_handoff(key, &restart_id)
|
||||
.await
|
||||
.expect("replacement handoff is acknowledged");
|
||||
|
||||
turn.finish();
|
||||
assert_eq!(input.busy_state(key), domain::input::AgentBusyState::Idle);
|
||||
assert_eq!(mailbox.pending(&key), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn human_turn_surfaces_explicit_restart_failure_and_releases_busy() {
|
||||
let mailbox = Arc::new(InMemoryMailbox::new());
|
||||
let input = Arc::new(MediatedInbox::new(
|
||||
Arc::clone(&mailbox),
|
||||
Arc::new(SystemMillisClock),
|
||||
));
|
||||
let coordinator = Arc::new(application::RestartRendezvousCoordinator::new());
|
||||
let project_id = ProjectId::from_uuid(Uuid::from_u128(290));
|
||||
let agent_id = AgentId::from_uuid(Uuid::from_u128(2));
|
||||
let key = domain::RuntimeAgentKey::new(project_id, agent_id);
|
||||
let mut turn = StructuredHumanTurnGuard::start(
|
||||
Arc::clone(&input) as Arc<dyn domain::input::InputMediator>,
|
||||
Arc::clone(&mailbox) as Arc<dyn domain::mailbox::AgentMailbox>,
|
||||
Arc::clone(&coordinator),
|
||||
project_id,
|
||||
agent_id,
|
||||
"restart me",
|
||||
);
|
||||
let restart_id = "restart-human-failure-290".to_owned();
|
||||
|
||||
assert!(coordinator.begin(key, restart_id.clone()));
|
||||
coordinator.fail(key, &restart_id, "replacement failed".to_owned());
|
||||
assert_eq!(
|
||||
turn.transition_after_provider_terminal_async().await,
|
||||
StructuredHumanTurnTransition::Failed("replacement failed".to_owned())
|
||||
);
|
||||
|
||||
turn.finish();
|
||||
assert_eq!(input.busy_state(key), domain::input::AgentBusyState::Idle);
|
||||
assert_eq!(mailbox.pending(&key), 0);
|
||||
}
|
||||
}
|
||||
|
||||
impl AgentSelfRestartScheduler for AppAgentSelfRestarter {
|
||||
@ -1306,6 +1378,7 @@ impl AgentSelfRestartScheduler for AppAgentSelfRestarter {
|
||||
let turn = StructuredHumanTurnGuard::start(
|
||||
Arc::clone(&input),
|
||||
Arc::clone(&mailbox),
|
||||
Arc::clone(&rendezvous),
|
||||
project.id,
|
||||
agent,
|
||||
application::SELF_RESTART_CONTINUATION_PROMPT,
|
||||
@ -1361,23 +1434,39 @@ impl AgentSelfRestartScheduler for AppAgentSelfRestarter {
|
||||
///
|
||||
/// Direct chat adapters (`agent_send`, web `chat.send`) still drive the provider
|
||||
/// through `AgentSession::send`, but they must share the same busy/FIFO accounting
|
||||
/// as structured inter-agent asks. This guard starts a silent human turn and
|
||||
/// retires it on terminal outcome or drop.
|
||||
/// as structured inter-agent asks. This guard starts a silent human turn,
|
||||
/// registers it for restart handoff, and retires it only on the logical terminal
|
||||
/// outcome or drop.
|
||||
pub struct StructuredHumanTurnGuard {
|
||||
input: Arc<dyn domain::input::InputMediator>,
|
||||
mailbox: Arc<dyn domain::mailbox::AgentMailbox>,
|
||||
agent: domain::RuntimeAgentKey,
|
||||
ticket_id: domain::mailbox::TicketId,
|
||||
armed: Arc<std::sync::atomic::AtomicBool>,
|
||||
restart: Option<application::RestartRendezvousRegistration>,
|
||||
finish_on_drop: bool,
|
||||
}
|
||||
|
||||
/// Control-plane outcome observed when one concrete provider turn terminates.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum StructuredHumanTurnTransition {
|
||||
/// No self-restart owns this terminal event; it is authoritative for the UI.
|
||||
Complete,
|
||||
/// The old terminal event is superseded; continue on the replacement runtime.
|
||||
Resume,
|
||||
/// Replacement failed explicitly.
|
||||
Failed(String),
|
||||
/// The user cancelled the logical turn while replacement was in flight.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl StructuredHumanTurnGuard {
|
||||
/// Starts the mediated busy turn for a direct human structured chat message.
|
||||
#[must_use]
|
||||
pub fn start(
|
||||
input: Arc<dyn domain::input::InputMediator>,
|
||||
mailbox: Arc<dyn domain::mailbox::AgentMailbox>,
|
||||
restart_rendezvous: Arc<application::RestartRendezvousCoordinator>,
|
||||
project_id: ProjectId,
|
||||
agent_id: AgentId,
|
||||
prompt: &str,
|
||||
@ -1397,6 +1486,7 @@ impl StructuredHumanTurnGuard {
|
||||
agent,
|
||||
ticket_id,
|
||||
armed: Arc::new(std::sync::atomic::AtomicBool::new(owns_turn)),
|
||||
restart: owns_turn.then(|| restart_rendezvous.register(agent)),
|
||||
finish_on_drop: true,
|
||||
}
|
||||
}
|
||||
@ -1410,10 +1500,55 @@ impl StructuredHumanTurnGuard {
|
||||
agent: self.agent,
|
||||
ticket_id: self.ticket_id,
|
||||
armed: Arc::clone(&self.armed),
|
||||
restart: None,
|
||||
finish_on_drop: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves whether a concrete provider terminal event really ends the
|
||||
/// logical human turn. During self-restart it waits for the replacement
|
||||
/// session, then acknowledges the handoff so the scheduler can complete.
|
||||
pub async fn transition_after_provider_terminal_async(
|
||||
&mut self,
|
||||
) -> StructuredHumanTurnTransition {
|
||||
let Some(restart) = self.restart.as_mut() else {
|
||||
return StructuredHumanTurnTransition::Complete;
|
||||
};
|
||||
loop {
|
||||
match restart.state() {
|
||||
application::RestartRendezvousState::Idle => {
|
||||
return StructuredHumanTurnTransition::Complete;
|
||||
}
|
||||
application::RestartRendezvousState::Restarting { .. } => {
|
||||
if restart.changed().await.is_err() {
|
||||
return StructuredHumanTurnTransition::Failed(
|
||||
"restart coordination closed".to_owned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
application::RestartRendezvousState::Ready { restart_id } => {
|
||||
restart.accept_ready(&restart_id);
|
||||
return StructuredHumanTurnTransition::Resume;
|
||||
}
|
||||
application::RestartRendezvousState::Failed { message, .. } => {
|
||||
return StructuredHumanTurnTransition::Failed(message);
|
||||
}
|
||||
application::RestartRendezvousState::Cancelled { .. } => {
|
||||
return StructuredHumanTurnTransition::Cancelled;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports whether a self-restart already owns the current concrete
|
||||
/// provider failure. The scheduler publishes `Restarting` synchronously
|
||||
/// before the MCP tool returns, so this check does not inspect provider text.
|
||||
pub fn restart_is_active(&mut self) -> bool {
|
||||
self.restart.as_mut().is_some_and(|restart| {
|
||||
!matches!(restart.state(), application::RestartRendezvousState::Idle)
|
||||
})
|
||||
}
|
||||
|
||||
/// Records proof of liveness for the running turn.
|
||||
pub fn mark_alive(&self) {
|
||||
if self.armed.load(std::sync::atomic::Ordering::Acquire) {
|
||||
@ -2005,6 +2140,8 @@ pub struct BackendCore {
|
||||
/// Mailbox paired with [`Self::turn_watch_input`], used by direct structured
|
||||
/// human turns to retire the synthetic busy/FIFO ticket on terminal outcome.
|
||||
pub input_mailbox: Arc<dyn domain::mailbox::AgentMailbox>,
|
||||
/// Stable restart control-plane shared by delegated and direct human turns.
|
||||
pub restart_rendezvous: Arc<application::RestartRendezvousCoordinator>,
|
||||
/// Handles des watches de fin-de-tour vivants, par clé runtime. (Re)lancer un agent
|
||||
/// dans un projet **remplace** son handle (l'ancien est droppé ⇒ polling arrêté) ;
|
||||
/// fermer/arrêter l'agent le retire. `Mutex` car launch/stop y accèdent concurremment.
|
||||
@ -3639,7 +3776,7 @@ impl BackendCore {
|
||||
.with_conversations(Arc::clone(&conversation_registry))
|
||||
.with_events(Arc::clone(&events_port))
|
||||
.with_self_restart(self_restart)
|
||||
.with_restart_rendezvous(restart_rendezvous)
|
||||
.with_restart_rendezvous(Arc::clone(&restart_rendezvous))
|
||||
// Faits OS/runtime (exe $APPIMAGE/current_exe + endpoint loopback) pour
|
||||
// que les (re)lancements issus du chemin `ask` écrivent la déclaration MCP
|
||||
// réelle ⇒ le pont MCP de la cible se spawne et `idea_reply` débloque le round-trip.
|
||||
@ -3757,7 +3894,8 @@ impl BackendCore {
|
||||
Arc::clone(&orchestrator_service),
|
||||
)
|
||||
.with_execution_cascade(Arc::clone(&orchestrator_service)
|
||||
as Arc<dyn application::StopAgentExecutionCascade>),
|
||||
as Arc<dyn application::StopAgentExecutionCascade>)
|
||||
.with_restart_rendezvous(Arc::clone(&restart_rendezvous)),
|
||||
);
|
||||
|
||||
// --- Windows (L10) ---
|
||||
@ -3952,6 +4090,7 @@ impl BackendCore {
|
||||
turn_watcher,
|
||||
turn_watch_input: Arc::clone(&input_mediator),
|
||||
input_mailbox: Arc::clone(&mailbox),
|
||||
restart_rendezvous: Arc::clone(&restart_rendezvous),
|
||||
turn_watch_handles: Mutex::new(HashMap::new()),
|
||||
fs_port: Arc::clone(&fs_port),
|
||||
home_dir,
|
||||
|
||||
@ -1882,6 +1882,7 @@ async fn ws_send_chat(
|
||||
backend::StructuredHumanTurnGuard::start(
|
||||
Arc::clone(&state.app.turn_watch_input),
|
||||
Arc::clone(&state.app.input_mailbox),
|
||||
Arc::clone(&state.app.restart_rendezvous),
|
||||
*project_id,
|
||||
*agent_id,
|
||||
&prompt,
|
||||
@ -1898,6 +1899,9 @@ async fn ws_send_chat(
|
||||
guard.mark_alive();
|
||||
}
|
||||
}
|
||||
if matches!(event, ReplyEvent::Final { .. } | ReplyEvent::Error { .. }) {
|
||||
continue;
|
||||
}
|
||||
let chunk = reply_event_chunk(event);
|
||||
if tap_out
|
||||
.blocking_send(ServerFrame::chat_output(None, sid, chunk))
|
||||
@ -1909,35 +1913,128 @@ async fn ws_send_chat(
|
||||
});
|
||||
let stream_result = session.send_with_tap(&prompt, tap_tx).await;
|
||||
let _ = tap_pump.join();
|
||||
let stream = stream_result.map_err(|err| ErrorDto {
|
||||
code: "CHAT_SEND_FAILED".to_owned(),
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
let mut turn_guard = turn_guard;
|
||||
let mut stream = match stream_result {
|
||||
Ok(stream) => stream,
|
||||
Err(_err)
|
||||
if turn_guard
|
||||
.as_mut()
|
||||
.is_some_and(|guard| guard.restart_is_active()) =>
|
||||
{
|
||||
Box::new(std::iter::empty()) as domain::ports::ReplyStream
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(ErrorDto {
|
||||
code: "CHAT_SEND_FAILED".to_owned(),
|
||||
message: err.to_string(),
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
let mut sent_ack = false;
|
||||
for event in stream {
|
||||
if !matches!(event, ReplyEvent::Final { .. }) {
|
||||
if let Some(guard) = &turn_guard {
|
||||
guard.mark_alive();
|
||||
'logical_turn: loop {
|
||||
let mut concrete_terminal = false;
|
||||
let mut resume = false;
|
||||
for event in stream {
|
||||
let provider_terminal =
|
||||
matches!(event, ReplyEvent::Final { .. } | ReplyEvent::Error { .. });
|
||||
if provider_terminal {
|
||||
concrete_terminal = true;
|
||||
if let Some(guard) = turn_guard.as_mut() {
|
||||
match guard.transition_after_provider_terminal_async().await {
|
||||
backend::StructuredHumanTurnTransition::Complete => {}
|
||||
backend::StructuredHumanTurnTransition::Resume => {
|
||||
resume = true;
|
||||
break;
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Failed(message) => {
|
||||
let chunk = reply_event_chunk(ReplyEvent::Error { message });
|
||||
tx.send(ServerFrame::chat_output(
|
||||
(!sent_ack).then_some(frame.id.as_str()),
|
||||
sid,
|
||||
chunk,
|
||||
))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
sent_ack = true;
|
||||
break 'logical_turn;
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Cancelled => {
|
||||
break 'logical_turn;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if let Some(guard) = &turn_guard {
|
||||
guard.mark_alive();
|
||||
}
|
||||
}
|
||||
let chunk = reply_event_chunk(event);
|
||||
if !sent_ack {
|
||||
tx.send(ServerFrame::chat_output(Some(&frame.id), sid, chunk))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
sent_ack = true;
|
||||
} else {
|
||||
tx.send(ServerFrame::chat_output(None, sid, chunk))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let chunk = reply_event_chunk(event);
|
||||
if !sent_ack {
|
||||
tx.send(ServerFrame::chat_output(Some(&frame.id), sid, chunk))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
sent_ack = true;
|
||||
} else {
|
||||
tx.send(ServerFrame::chat_output(None, sid, chunk))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
|
||||
if !resume && !concrete_terminal {
|
||||
if let Some(guard) = turn_guard.as_mut() {
|
||||
match guard.transition_after_provider_terminal_async().await {
|
||||
backend::StructuredHumanTurnTransition::Complete => {}
|
||||
backend::StructuredHumanTurnTransition::Resume => resume = true,
|
||||
backend::StructuredHumanTurnTransition::Failed(message) => {
|
||||
let chunk = reply_event_chunk(ReplyEvent::Error { message });
|
||||
tx.send(ServerFrame::chat_output(
|
||||
(!sent_ack).then_some(frame.id.as_str()),
|
||||
sid,
|
||||
chunk,
|
||||
))
|
||||
.await
|
||||
.map_err(|_| ErrorDto {
|
||||
code: "WS_CLOSED".to_owned(),
|
||||
message: "websocket output closed".to_owned(),
|
||||
})?;
|
||||
sent_ack = true;
|
||||
}
|
||||
backend::StructuredHumanTurnTransition::Cancelled => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
if !resume {
|
||||
break;
|
||||
}
|
||||
let Some((project_id, agent_id, _, _)) = session_meta.as_ref() else {
|
||||
break;
|
||||
};
|
||||
let replacement = state
|
||||
.app
|
||||
.structured_sessions
|
||||
.session_for_agent_in_project(*project_id, agent_id)
|
||||
.ok_or_else(|| ErrorDto {
|
||||
code: "CHAT_RESTART_FAILED".to_owned(),
|
||||
message: "restart completed without a replacement structured session".to_owned(),
|
||||
})?;
|
||||
stream = replacement
|
||||
.send(application::SELF_RESTART_CONTINUATION_PROMPT)
|
||||
.await
|
||||
.map_err(|err| ErrorDto {
|
||||
code: "CHAT_RESTART_FAILED".to_owned(),
|
||||
message: err.to_string(),
|
||||
})?;
|
||||
}
|
||||
|
||||
if !sent_ack {
|
||||
|
||||
Reference in New Issue
Block a user