feat(infrastructure): store, sink de complétion et mailbox des tâches de fond (B2-B4)
Adapters de persistance des BackgroundTask (store), sink de récupération de complétion post-tour et boîte de réception (mailbox) pour les messages concurrents, câblés sur l'entrée. Couvert par background_task_store, background_completion_sink, agent_inbox et orchestrator_watcher (verts). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
233
crates/infrastructure/src/background_task.rs
Normal file
233
crates/infrastructure/src/background_task.rs
Normal file
@ -0,0 +1,233 @@
|
||||
//! Background task completion sink.
|
||||
//!
|
||||
//! The sink is the durable boundary between a runner completion and later
|
||||
//! mailbox/wake work: it writes the terminal task state to the
|
||||
//! [`BackgroundTaskStore`](domain::ports::BackgroundTaskStore) first, then emits
|
||||
//! a lightweight "ready to deliver" signal. It never wakes an agent and never
|
||||
//! enqueues mailbox items; B4/B5 consume the ready signal.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use thiserror::Error;
|
||||
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use domain::ports::{BackgroundTaskCompletion, BackgroundTaskRunner, BackgroundTaskStore};
|
||||
use domain::{
|
||||
AgentInbox, BackgroundTaskPortError, InboxError, InboxItem, InboxItemKind, InboxReceiptStatus,
|
||||
InboxSource, ProjectId, TaskId,
|
||||
};
|
||||
|
||||
/// Signal emitted only after a completion has been persisted.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackgroundTaskReadyToDeliver {
|
||||
/// Persisted task id.
|
||||
pub task_id: TaskId,
|
||||
/// Owning project, copied from the persisted task.
|
||||
pub project_id: ProjectId,
|
||||
/// Agent that owns the completion delivery.
|
||||
pub owner_agent_id: domain::AgentId,
|
||||
}
|
||||
|
||||
/// Outcome of processing one completion event.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum BackgroundCompletionSinkOutcome {
|
||||
/// Completion was persisted and signalled as ready to deliver.
|
||||
PersistedAndSignalled(BackgroundTaskReadyToDeliver),
|
||||
/// Task was already terminal, so the duplicate completion was ignored.
|
||||
IgnoredAlreadyTerminal {
|
||||
/// Ignored task id.
|
||||
task_id: TaskId,
|
||||
},
|
||||
/// Another completion for this task was already processed by this sink.
|
||||
IgnoredDuplicate {
|
||||
/// Ignored task id.
|
||||
task_id: TaskId,
|
||||
},
|
||||
/// The task no longer exists in the store.
|
||||
IgnoredMissingTask {
|
||||
/// Ignored task id.
|
||||
task_id: TaskId,
|
||||
},
|
||||
}
|
||||
|
||||
/// Errors raised by the completion sink.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
pub enum BackgroundCompletionSinkError {
|
||||
/// Store operation failed.
|
||||
#[error("background completion store failed: {0}")]
|
||||
Store(#[from] BackgroundTaskPortError),
|
||||
/// The ready-to-deliver channel is closed.
|
||||
#[error("background completion ready signal channel is closed")]
|
||||
ReadySignalClosed,
|
||||
}
|
||||
|
||||
/// Consumes runner completions, persists terminal state, then signals delivery.
|
||||
pub struct BackgroundCompletionSink {
|
||||
store: Arc<dyn BackgroundTaskStore>,
|
||||
ready: UnboundedSender<BackgroundTaskReadyToDeliver>,
|
||||
processed: Mutex<HashSet<TaskId>>,
|
||||
}
|
||||
|
||||
/// Handle for the ready-to-inbox bridge task.
|
||||
pub type BackgroundReadyInboxBridgeHandle = JoinHandle<()>;
|
||||
|
||||
impl BackgroundCompletionSink {
|
||||
/// Builds a sink from a task store and ready-to-deliver channel.
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
store: Arc<dyn BackgroundTaskStore>,
|
||||
ready: UnboundedSender<BackgroundTaskReadyToDeliver>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
ready,
|
||||
processed: Mutex::new(HashSet::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts consuming [`BackgroundTaskRunner::subscribe_completions`].
|
||||
///
|
||||
/// The domain completion stream is a blocking iterator, so consumption runs on
|
||||
/// a blocking task and re-enters the current Tokio runtime for persistence.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if called outside a Tokio runtime.
|
||||
#[must_use]
|
||||
pub fn start_from_runner(
|
||||
self: Arc<Self>,
|
||||
runner: Arc<dyn BackgroundTaskRunner>,
|
||||
) -> JoinHandle<()> {
|
||||
let mut stream = runner.subscribe_completions();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
while let Some(completion) = stream.next() {
|
||||
let _ = runtime.block_on(self.process_completion(completion));
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Processes a single completion event.
|
||||
///
|
||||
/// Order is strict: `store.save(terminal_task)` must succeed before the ready
|
||||
/// signal is sent. If persistence fails, no ready signal is emitted and the
|
||||
/// task id is released so a later retry can persist it.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`BackgroundCompletionSinkError`] if the store fails, the task transition
|
||||
/// is invalid, or the ready channel is closed after persistence.
|
||||
pub async fn process_completion(
|
||||
&self,
|
||||
completion: BackgroundTaskCompletion,
|
||||
) -> Result<BackgroundCompletionSinkOutcome, BackgroundCompletionSinkError> {
|
||||
{
|
||||
let mut processed = self.processed.lock().await;
|
||||
if processed.contains(&completion.task_id) {
|
||||
return Ok(BackgroundCompletionSinkOutcome::IgnoredDuplicate {
|
||||
task_id: completion.task_id,
|
||||
});
|
||||
}
|
||||
processed.insert(completion.task_id);
|
||||
}
|
||||
|
||||
let outcome = self.persist_and_signal(completion.clone()).await;
|
||||
if outcome.is_err() {
|
||||
self.processed.lock().await.remove(&completion.task_id);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn persist_and_signal(
|
||||
&self,
|
||||
completion: BackgroundTaskCompletion,
|
||||
) -> Result<BackgroundCompletionSinkOutcome, BackgroundCompletionSinkError> {
|
||||
let Some(task) = self.store.get(completion.task_id).await? else {
|
||||
return Ok(BackgroundCompletionSinkOutcome::IgnoredMissingTask {
|
||||
task_id: completion.task_id,
|
||||
});
|
||||
};
|
||||
if task.is_terminal() {
|
||||
return Ok(BackgroundCompletionSinkOutcome::IgnoredAlreadyTerminal {
|
||||
task_id: completion.task_id,
|
||||
});
|
||||
}
|
||||
|
||||
let terminal = task.complete(completion.result).map_err(|e| {
|
||||
BackgroundCompletionSinkError::Store(BackgroundTaskPortError::Invalid(e.to_string()))
|
||||
})?;
|
||||
self.store.save(&terminal).await?;
|
||||
|
||||
let ready = BackgroundTaskReadyToDeliver {
|
||||
task_id: terminal.id,
|
||||
project_id: terminal.project_id,
|
||||
owner_agent_id: terminal.owner_agent_id,
|
||||
};
|
||||
self.ready
|
||||
.send(ready.clone())
|
||||
.map_err(|_| BackgroundCompletionSinkError::ReadySignalClosed)?;
|
||||
Ok(BackgroundCompletionSinkOutcome::PersistedAndSignalled(
|
||||
ready,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the B3→B4 bridge: persisted completions become inbox items.
|
||||
///
|
||||
/// Overflow of completion/system items is intentionally non-fatal: the completion
|
||||
/// is already durable and remains delivery-pending for boot reconcile.
|
||||
#[must_use]
|
||||
pub fn start_background_ready_inbox_bridge(
|
||||
mut ready: UnboundedReceiver<BackgroundTaskReadyToDeliver>,
|
||||
inbox: Arc<dyn AgentInbox>,
|
||||
) -> BackgroundReadyInboxBridgeHandle {
|
||||
tokio::spawn(async move {
|
||||
while let Some(ready) = ready.recv().await {
|
||||
let item = InboxItem {
|
||||
id: domain::TicketId::new_random(),
|
||||
agent_id: ready.owner_agent_id,
|
||||
source: InboxSource::BackgroundTask {
|
||||
task_id: ready.task_id,
|
||||
},
|
||||
kind: InboxItemKind::BackgroundCompletion,
|
||||
body: format!("Background task {} completed.", ready.task_id),
|
||||
created_at_ms: now_ms(),
|
||||
correlation_id: Some(ready.task_id.to_string()),
|
||||
};
|
||||
match inbox.enqueue_message(ready.owner_agent_id, item) {
|
||||
Ok(receipt) if receipt.status == InboxReceiptStatus::Deferred => {
|
||||
application::diag!(
|
||||
"[background-task] completion deferred: task={} owner={} queue_depth={}",
|
||||
ready.task_id,
|
||||
ready.owner_agent_id,
|
||||
receipt.depth
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(InboxError::InboxFull { .. }) => {
|
||||
application::diag!(
|
||||
"[background-task] completion inbox full but durable: task={} owner={}",
|
||||
ready.task_id,
|
||||
ready.owner_agent_id
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
application::diag!(
|
||||
"[background-task] completion inbox enqueue failed: task={} owner={} err={err}",
|
||||
ready.task_id,
|
||||
ready.owner_agent_id
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn now_ms() -> u64 {
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
@ -26,8 +26,12 @@ use std::time::Duration;
|
||||
|
||||
use domain::events::DomainEvent;
|
||||
use domain::ids::AgentId;
|
||||
use domain::inbox::{
|
||||
AgentInbox, AgentInboxSnapshot, InboxError, InboxItem, InboxReceipt, InboxReceiptStatus,
|
||||
InboxSource, DEFAULT_AGENT_INBOX_CAPACITY,
|
||||
};
|
||||
use domain::input::{AgentBusyState, AgentLiveness, InputMediator, SubmitConfig};
|
||||
use domain::mailbox::{AgentMailbox, PendingReply, Ticket, TicketId};
|
||||
use domain::mailbox::{AgentMailbox, AgentQueueSnapshot, PendingReply, Ticket, TicketId};
|
||||
use domain::ports::{EventBus, PtyHandle, PtyPort};
|
||||
|
||||
use crate::mailbox::InMemoryMailbox;
|
||||
@ -496,6 +500,11 @@ pub struct MediatedInbox {
|
||||
/// `set_stall_threshold` and consumed to arm a fresh liveness window on the enqueue
|
||||
/// that starts a turn. Absent ⇒ `None` (no stall detection — legacy behaviour).
|
||||
stall: Mutex<HashMap<AgentId, Option<u32>>>,
|
||||
/// Rich inbox payloads keyed by the mailbox ticket id. The mailbox remains the
|
||||
/// only FIFO; this map is metadata only.
|
||||
inbox_items: Mutex<HashMap<TicketId, InboxItem>>,
|
||||
/// Bounded inbox capacity per agent.
|
||||
inbox_capacity: usize,
|
||||
/// Fenêtre de grâce (G) propagée au [`BusyTracker`] à chaque (re)construction.
|
||||
/// Prod = [`PROMPT_READY_GRACE`] (2 s) ; surchargée par [`Self::with_grace`] en test.
|
||||
grace: Duration,
|
||||
@ -525,6 +534,8 @@ impl MediatedInbox {
|
||||
front_owned,
|
||||
submit: Mutex::new(HashMap::new()),
|
||||
stall: Mutex::new(HashMap::new()),
|
||||
inbox_items: Mutex::new(HashMap::new()),
|
||||
inbox_capacity: configured_inbox_capacity(),
|
||||
grace: PROMPT_READY_GRACE,
|
||||
}
|
||||
}
|
||||
@ -716,6 +727,8 @@ impl MediatedInbox {
|
||||
front_owned,
|
||||
submit: Mutex::new(HashMap::new()),
|
||||
stall: Mutex::new(HashMap::new()),
|
||||
inbox_items: Mutex::new(HashMap::new()),
|
||||
inbox_capacity: configured_inbox_capacity(),
|
||||
grace: PROMPT_READY_GRACE,
|
||||
}
|
||||
}
|
||||
@ -729,6 +742,13 @@ impl MediatedInbox {
|
||||
)
|
||||
}
|
||||
|
||||
/// Overrides bounded inbox capacity. Intended for tests and explicit wiring.
|
||||
#[must_use]
|
||||
pub fn with_inbox_capacity(mut self, capacity: usize) -> Self {
|
||||
self.inbox_capacity = capacity;
|
||||
self
|
||||
}
|
||||
|
||||
fn handles(&self) -> std::sync::MutexGuard<'_, HashMap<AgentId, PtyHandle>> {
|
||||
self.handles
|
||||
.lock()
|
||||
@ -747,6 +767,12 @@ impl MediatedInbox {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn inbox_items(&self) -> std::sync::MutexGuard<'_, HashMap<TicketId, InboxItem>> {
|
||||
self.inbox_items
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// **Détection de stagnation** (lot 2) : balaie les agents `Busy` et bascule
|
||||
/// `Alive→Stalled` ceux dont le dernier battement remonte à plus de
|
||||
/// `stall_after_ms`. Émet `AgentLivenessChanged` **une fois par transition**. La
|
||||
@ -1023,6 +1049,107 @@ impl InputMediator for MediatedInbox {
|
||||
}
|
||||
}
|
||||
|
||||
impl AgentInbox for MediatedInbox {
|
||||
fn enqueue_message(&self, agent: AgentId, item: InboxItem) -> Result<InboxReceipt, InboxError> {
|
||||
if item.agent_id != agent {
|
||||
return Err(InboxError::AgentMismatch {
|
||||
agent_id: agent,
|
||||
item_agent_id: item.agent_id,
|
||||
});
|
||||
}
|
||||
|
||||
let depth_before = self.mailbox.pending(&agent);
|
||||
if depth_before >= self.inbox_capacity {
|
||||
if item.is_lossless_system() {
|
||||
return Ok(InboxReceipt {
|
||||
item_id: item.id,
|
||||
agent_id: agent,
|
||||
depth: depth_before,
|
||||
status: InboxReceiptStatus::Deferred,
|
||||
});
|
||||
}
|
||||
return Err(InboxError::InboxFull {
|
||||
agent_id: agent,
|
||||
capacity: self.inbox_capacity,
|
||||
});
|
||||
}
|
||||
|
||||
let item_id = item.id;
|
||||
let ticket = inbox_item_to_ticket(&item);
|
||||
self.inbox_items().insert(item_id, item);
|
||||
let _pending = match ticket.source {
|
||||
domain::input::InputSource::Human => self.enqueue(agent, ticket),
|
||||
domain::input::InputSource::Agent { .. } => self.enqueue_silent(agent, ticket),
|
||||
};
|
||||
let depth = self.mailbox.pending(&agent);
|
||||
if let Some(events) = &self.tracker.events {
|
||||
events.publish(DomainEvent::AgentInboxQueued {
|
||||
agent_id: agent,
|
||||
depth,
|
||||
});
|
||||
}
|
||||
Ok(InboxReceipt {
|
||||
item_id,
|
||||
agent_id: agent,
|
||||
depth,
|
||||
status: InboxReceiptStatus::Queued,
|
||||
})
|
||||
}
|
||||
|
||||
fn dequeue_next(&self, agent: AgentId) -> Option<InboxItem> {
|
||||
let head = self.mailbox.head_ticket(&agent)?;
|
||||
let item = self.inbox_items().remove(&head)?;
|
||||
self.mailbox.cancel_head(agent, head);
|
||||
if let Some(events) = &self.tracker.events {
|
||||
events.publish(DomainEvent::AgentInboxDrained {
|
||||
agent_id: agent,
|
||||
depth: self.snapshot(agent).depth,
|
||||
});
|
||||
}
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn snapshot(&self, agent: AgentId) -> AgentInboxSnapshot {
|
||||
let items_by_id = self.inbox_items();
|
||||
let items: Vec<InboxItem> = self
|
||||
.mailbox
|
||||
.queue_for(agent)
|
||||
.into_iter()
|
||||
.filter_map(|ticket| items_by_id.get(&ticket.id).cloned())
|
||||
.collect();
|
||||
AgentInboxSnapshot {
|
||||
agent_id: agent,
|
||||
depth: self.mailbox.pending(&agent),
|
||||
items,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn configured_inbox_capacity() -> usize {
|
||||
std::env::var("IDEA_AGENT_INBOX_CAPACITY")
|
||||
.ok()
|
||||
.and_then(|raw| raw.parse::<usize>().ok())
|
||||
.filter(|capacity| *capacity > 0)
|
||||
.unwrap_or(DEFAULT_AGENT_INBOX_CAPACITY)
|
||||
}
|
||||
|
||||
fn inbox_item_to_ticket(item: &InboxItem) -> Ticket {
|
||||
let conversation = domain::conversation::ConversationId::from_uuid(uuid::Uuid::nil());
|
||||
match item.source {
|
||||
InboxSource::Agent { agent_id } => Ticket::from_agent(
|
||||
item.id,
|
||||
agent_id,
|
||||
conversation,
|
||||
agent_id.to_string(),
|
||||
item.body.clone(),
|
||||
),
|
||||
InboxSource::Human => Ticket::from_human(item.id, conversation, "vous", item.body.clone()),
|
||||
InboxSource::BackgroundTask { .. } | InboxSource::System => {
|
||||
Ticket::from_human(item.id, conversation, "IdeA", item.body.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![warn(missing_docs)]
|
||||
|
||||
pub mod background_task;
|
||||
pub mod clock;
|
||||
pub mod conversation;
|
||||
pub mod conversation_log;
|
||||
@ -36,6 +37,11 @@ pub mod session;
|
||||
pub mod store;
|
||||
pub mod timeparse;
|
||||
|
||||
pub use background_task::{
|
||||
start_background_ready_inbox_bridge, BackgroundCompletionSink, BackgroundCompletionSinkError,
|
||||
BackgroundCompletionSinkOutcome, BackgroundReadyInboxBridgeHandle,
|
||||
BackgroundTaskReadyToDeliver,
|
||||
};
|
||||
pub use clock::SystemClock;
|
||||
pub use conversation::InMemoryConversationRegistry;
|
||||
pub use conversation_log::{
|
||||
@ -74,9 +80,10 @@ pub use store::OnnxEmbedder;
|
||||
pub use store::{detect_ollama, HttpEmbedder, DEFAULT_LOCAL_EMBED_ENDPOINT};
|
||||
pub use store::{
|
||||
embedder_from_profile, index_token_size, onnx_model_is_cached, should_use_vector,
|
||||
AdaptiveMemoryRecall, EmbedderEnvProbe, FsEmbedderProfileStore, FsEmbedderPromptStore,
|
||||
FsLiveStateStore, FsMemoryStore, FsPermissionStore, FsProfileStore, FsProjectStore,
|
||||
FsSkillStore, FsTemplateStore, HashEmbedder, IdeaiContextStore, NaiveMemoryRecall,
|
||||
OnnxModelInfo, StubEmbedder, VectorMemoryRecall, DEFAULT_OLLAMA_BASE_URL, ONNX_CACHE_SUBDIR,
|
||||
RECOMMENDED_ONNX_MODELS, VECTOR_HTTP_ENABLED, VECTOR_ONNX_ENABLED,
|
||||
AdaptiveMemoryRecall, BackgroundTaskReconcileReport, EmbedderEnvProbe, FsBackgroundTaskStore,
|
||||
FsEmbedderProfileStore, FsEmbedderPromptStore, FsLiveStateStore, FsMemoryStore,
|
||||
FsPermissionStore, FsProfileStore, FsProjectStore, FsSkillStore, FsTemplateStore, HashEmbedder,
|
||||
IdeaiContextStore, NaiveMemoryRecall, OnnxModelInfo, StubEmbedder, VectorMemoryRecall,
|
||||
DEFAULT_OLLAMA_BASE_URL, ONNX_CACHE_SUBDIR, RECOMMENDED_ONNX_MODELS, VECTOR_HTTP_ENABLED,
|
||||
VECTOR_ONNX_ENABLED,
|
||||
};
|
||||
|
||||
390
crates/infrastructure/src/store/background_task.rs
Normal file
390
crates/infrastructure/src/store/background_task.rs
Normal file
@ -0,0 +1,390 @@
|
||||
//! [`FsBackgroundTaskStore`] — durable store for first-class background tasks.
|
||||
//!
|
||||
//! Persistence is segmented by project id under the project root:
|
||||
//!
|
||||
//! ```text
|
||||
//! <project_root>/.ideai/background-tasks/<projectId>.json
|
||||
//! ```
|
||||
//!
|
||||
//! Each file is a small JSON document `{ version, projectId, tasks }`. Writes are
|
||||
//! atomic: serialize to `<projectId>.json.tmp`, then rename over the target. The
|
||||
//! in-memory registry indexes open tasks by owner agent and task id; it is
|
||||
//! rebuilt lazily from disk on first use and updated on every successful mutation.
|
||||
//!
|
||||
//! Boot reconcile rule for B2: because the runner registry is not wired yet, a
|
||||
//! task in `Running` or `Waiting` whose id is not present in the caller-provided
|
||||
//! live handle list is marked `Failed` with a synthetic restart-loss error and
|
||||
//! `completion_delivered = false`. Existing terminal tasks that are not delivered
|
||||
//! are only reported as delivery-pending. `Queued` tasks are left unchanged.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use domain::{
|
||||
AgentId, BackgroundTask, BackgroundTaskPortError, BackgroundTaskResult, BackgroundTaskState,
|
||||
BackgroundTaskStore, ProjectId, ProjectPath, TaskId,
|
||||
};
|
||||
|
||||
const IDEAI_DIR: &str = ".ideai";
|
||||
const BACKGROUND_TASKS_DIR: &str = "background-tasks";
|
||||
const TASK_DOC_VERSION: u32 = 1;
|
||||
|
||||
/// File-backed implementation of [`BackgroundTaskStore`].
|
||||
pub struct FsBackgroundTaskStore {
|
||||
dir: PathBuf,
|
||||
registry: RwLock<RegistryIndex>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct RegistryIndex {
|
||||
loaded: bool,
|
||||
task_to_project: HashMap<TaskId, ProjectId>,
|
||||
open_by_agent: HashMap<AgentId, BTreeMap<TaskId, BackgroundTask>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct StoreState {
|
||||
docs: BTreeMap<ProjectId, TaskDoc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct TaskDoc {
|
||||
version: u32,
|
||||
project_id: ProjectId,
|
||||
tasks: Vec<BackgroundTask>,
|
||||
}
|
||||
|
||||
/// Summary returned by [`FsBackgroundTaskStore::reconcile_boot`].
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BackgroundTaskReconcileReport {
|
||||
/// Open tasks converted to `Failed` because no live runtime handle exists.
|
||||
pub failed_task_ids: Vec<TaskId>,
|
||||
/// Terminal tasks whose completion remains to be delivered.
|
||||
pub delivery_pending_task_ids: Vec<TaskId>,
|
||||
}
|
||||
|
||||
impl FsBackgroundTaskStore {
|
||||
/// Builds the store for a project root.
|
||||
#[must_use]
|
||||
pub fn new(root: &ProjectPath) -> Self {
|
||||
let dir = PathBuf::from(root.as_str())
|
||||
.join(IDEAI_DIR)
|
||||
.join(BACKGROUND_TASKS_DIR);
|
||||
Self {
|
||||
dir,
|
||||
registry: RwLock::new(RegistryIndex::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `<root>/.ideai/background-tasks`.
|
||||
#[must_use]
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
/// Returns the open task ids known by the in-memory registry for `agent_id`.
|
||||
///
|
||||
/// This is intentionally a registry view: it does not read the filesystem.
|
||||
pub async fn registry_open_task_ids_for_agent(&self, agent_id: AgentId) -> Vec<TaskId> {
|
||||
self.registry
|
||||
.read()
|
||||
.await
|
||||
.open_by_agent
|
||||
.get(&agent_id)
|
||||
.map(|tasks| tasks.keys().copied().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Reconciles persisted tasks against live runtime handles at boot.
|
||||
///
|
||||
/// B2 rule: `Running`/`Waiting` tasks missing from `live_task_ids` are marked
|
||||
/// `Failed` with a synthetic restart-loss failure and become delivery-pending.
|
||||
/// Already-terminal undelivered tasks are reported as delivery-pending. `Queued`
|
||||
/// tasks are left open.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`BackgroundTaskPortError`] on I/O, serialization or invalid transition.
|
||||
pub async fn reconcile_boot(
|
||||
&self,
|
||||
live_task_ids: &[TaskId],
|
||||
now_ms: u64,
|
||||
) -> Result<BackgroundTaskReconcileReport, BackgroundTaskPortError> {
|
||||
let live: HashSet<TaskId> = live_task_ids.iter().copied().collect();
|
||||
let mut state = self.read_all_docs().await?;
|
||||
let mut report = BackgroundTaskReconcileReport::default();
|
||||
let mut changed_projects = BTreeSet::new();
|
||||
|
||||
for (project_id, doc) in &mut state.docs {
|
||||
for task in &mut doc.tasks {
|
||||
match task.state {
|
||||
BackgroundTaskState::Running | BackgroundTaskState::Waiting
|
||||
if !live.contains(&task.id) =>
|
||||
{
|
||||
let finished_at_ms = now_ms.max(task.updated_at_ms);
|
||||
let result = BackgroundTaskResult::Failure {
|
||||
finished_at_ms,
|
||||
exit_code: None,
|
||||
error: "background task lost its runtime handle during IdeA restart"
|
||||
.into(),
|
||||
stdout_tail: None,
|
||||
stderr_tail: None,
|
||||
};
|
||||
*task = task.complete(result).map_err(invalid_task)?;
|
||||
report.failed_task_ids.push(task.id);
|
||||
report.delivery_pending_task_ids.push(task.id);
|
||||
changed_projects.insert(*project_id);
|
||||
}
|
||||
_ if task.has_pending_completion_delivery() => {
|
||||
report.delivery_pending_task_ids.push(task.id);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for project_id in changed_projects {
|
||||
if let Some(doc) = state.docs.get(&project_id) {
|
||||
self.write_doc(doc).await?;
|
||||
}
|
||||
}
|
||||
self.replace_registry_from_state(&state).await;
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
async fn ensure_registry_loaded(&self) -> Result<(), BackgroundTaskPortError> {
|
||||
if self.registry.read().await.loaded {
|
||||
return Ok(());
|
||||
}
|
||||
let state = self.read_all_docs().await?;
|
||||
self.replace_registry_from_state(&state).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn replace_registry_from_state(&self, state: &StoreState) {
|
||||
let mut registry = self.registry.write().await;
|
||||
*registry = RegistryIndex::from_state(state);
|
||||
}
|
||||
|
||||
fn path_for_project(&self, project_id: ProjectId) -> PathBuf {
|
||||
self.dir.join(format!("{project_id}.json"))
|
||||
}
|
||||
|
||||
fn tmp_path_for_project(&self, project_id: ProjectId) -> PathBuf {
|
||||
self.dir.join(format!("{project_id}.json.tmp"))
|
||||
}
|
||||
|
||||
async fn read_all_docs(&self) -> Result<StoreState, BackgroundTaskPortError> {
|
||||
let mut state = StoreState::default();
|
||||
let mut entries = match tokio::fs::read_dir(&self.dir).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(state),
|
||||
Err(e) => return Err(io_error(e)),
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await.map_err(io_error)? {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|ext| ext.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
let doc = self.read_doc_path(&path).await?;
|
||||
state.docs.insert(doc.project_id, doc);
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
async fn read_doc(&self, project_id: ProjectId) -> Result<TaskDoc, BackgroundTaskPortError> {
|
||||
let path = self.path_for_project(project_id);
|
||||
match self.read_doc_path(&path).await {
|
||||
Ok(doc) => Ok(doc),
|
||||
Err(BackgroundTaskPortError::NotFound) => Ok(TaskDoc {
|
||||
version: TASK_DOC_VERSION,
|
||||
project_id,
|
||||
tasks: Vec::new(),
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_doc_path(&self, path: &Path) -> Result<TaskDoc, BackgroundTaskPortError> {
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(bytes) => serde_json::from_slice(&bytes)
|
||||
.map_err(|e| BackgroundTaskPortError::Store(e.to_string())),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
Err(BackgroundTaskPortError::NotFound)
|
||||
}
|
||||
Err(e) => Err(io_error(e)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_doc(&self, doc: &TaskDoc) -> Result<(), BackgroundTaskPortError> {
|
||||
tokio::fs::create_dir_all(&self.dir)
|
||||
.await
|
||||
.map_err(io_error)?;
|
||||
let bytes = serde_json::to_vec_pretty(doc)
|
||||
.map_err(|e| BackgroundTaskPortError::Store(e.to_string()))?;
|
||||
let tmp = self.tmp_path_for_project(doc.project_id);
|
||||
tokio::fs::write(&tmp, &bytes).await.map_err(io_error)?;
|
||||
tokio::fs::rename(&tmp, self.path_for_project(doc.project_id))
|
||||
.await
|
||||
.map_err(io_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_task_project(
|
||||
&self,
|
||||
task_id: TaskId,
|
||||
) -> Result<Option<ProjectId>, BackgroundTaskPortError> {
|
||||
self.ensure_registry_loaded().await?;
|
||||
Ok(self
|
||||
.registry
|
||||
.read()
|
||||
.await
|
||||
.task_to_project
|
||||
.get(&task_id)
|
||||
.copied())
|
||||
}
|
||||
|
||||
async fn update_registry_for_task(&self, task: &BackgroundTask) {
|
||||
let mut registry = self.registry.write().await;
|
||||
registry.loaded = true;
|
||||
registry.task_to_project.insert(task.id, task.project_id);
|
||||
for tasks in registry.open_by_agent.values_mut() {
|
||||
tasks.remove(&task.id);
|
||||
}
|
||||
if !task.is_terminal() {
|
||||
registry
|
||||
.open_by_agent
|
||||
.entry(task.owner_agent_id)
|
||||
.or_default()
|
||||
.insert(task.id, task.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RegistryIndex {
|
||||
fn from_state(state: &StoreState) -> Self {
|
||||
let mut registry = Self {
|
||||
loaded: true,
|
||||
task_to_project: HashMap::new(),
|
||||
open_by_agent: HashMap::new(),
|
||||
};
|
||||
for doc in state.docs.values() {
|
||||
for task in &doc.tasks {
|
||||
registry.task_to_project.insert(task.id, task.project_id);
|
||||
if !task.is_terminal() {
|
||||
registry
|
||||
.open_by_agent
|
||||
.entry(task.owner_agent_id)
|
||||
.or_default()
|
||||
.insert(task.id, task.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
registry
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackgroundTaskStore for FsBackgroundTaskStore {
|
||||
async fn create(&self, task: &BackgroundTask) -> Result<(), BackgroundTaskPortError> {
|
||||
self.ensure_registry_loaded().await?;
|
||||
if self
|
||||
.registry
|
||||
.read()
|
||||
.await
|
||||
.task_to_project
|
||||
.contains_key(&task.id)
|
||||
{
|
||||
return Err(BackgroundTaskPortError::AlreadyExists);
|
||||
}
|
||||
|
||||
let mut doc = self.read_doc(task.project_id).await?;
|
||||
if doc.tasks.iter().any(|existing| existing.id == task.id) {
|
||||
return Err(BackgroundTaskPortError::AlreadyExists);
|
||||
}
|
||||
doc.tasks.push(task.clone());
|
||||
self.write_doc(&doc).await?;
|
||||
self.update_registry_for_task(task).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get(&self, id: TaskId) -> Result<Option<BackgroundTask>, BackgroundTaskPortError> {
|
||||
let Some(project_id) = self.find_task_project(id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let doc = self.read_doc(project_id).await?;
|
||||
Ok(doc.tasks.into_iter().find(|task| task.id == id))
|
||||
}
|
||||
|
||||
async fn save(&self, task: &BackgroundTask) -> Result<(), BackgroundTaskPortError> {
|
||||
self.ensure_registry_loaded().await?;
|
||||
let mut doc = self.read_doc(task.project_id).await?;
|
||||
if let Some(slot) = doc.tasks.iter_mut().find(|existing| existing.id == task.id) {
|
||||
*slot = task.clone();
|
||||
} else {
|
||||
return Err(BackgroundTaskPortError::NotFound);
|
||||
}
|
||||
self.write_doc(&doc).await?;
|
||||
self.update_registry_for_task(task).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_open_for_agent(
|
||||
&self,
|
||||
agent_id: AgentId,
|
||||
) -> Result<Vec<BackgroundTask>, BackgroundTaskPortError> {
|
||||
self.ensure_registry_loaded().await?;
|
||||
let registry = self.registry.read().await;
|
||||
let mut tasks: Vec<BackgroundTask> = registry
|
||||
.open_by_agent
|
||||
.get(&agent_id)
|
||||
.map(|tasks| tasks.values().cloned().collect())
|
||||
.unwrap_or_default();
|
||||
tasks.sort_by_key(|task| (task.created_at_ms, task.id));
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn list_undelivered_completions(
|
||||
&self,
|
||||
) -> Result<Vec<BackgroundTask>, BackgroundTaskPortError> {
|
||||
let mut tasks: Vec<BackgroundTask> = self
|
||||
.read_all_docs()
|
||||
.await?
|
||||
.docs
|
||||
.into_values()
|
||||
.flat_map(|doc| doc.tasks)
|
||||
.filter(BackgroundTask::has_pending_completion_delivery)
|
||||
.collect();
|
||||
tasks.sort_by_key(|task| (task.updated_at_ms, task.id));
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn mark_completion_delivered(
|
||||
&self,
|
||||
task_id: TaskId,
|
||||
) -> Result<(), BackgroundTaskPortError> {
|
||||
let task = self
|
||||
.get(task_id)
|
||||
.await?
|
||||
.ok_or(BackgroundTaskPortError::NotFound)?;
|
||||
let delivered = match task.mark_completion_delivered() {
|
||||
Ok(delivered) => delivered,
|
||||
Err(domain::BackgroundTaskError::CompletionAlreadyDelivered) => return Ok(()),
|
||||
Err(e) => return Err(invalid_task(e)),
|
||||
};
|
||||
self.save(&delivered).await
|
||||
}
|
||||
}
|
||||
|
||||
fn io_error(error: std::io::Error) -> BackgroundTaskPortError {
|
||||
BackgroundTaskPortError::Store(error.to_string())
|
||||
}
|
||||
|
||||
fn invalid_task(error: domain::BackgroundTaskError) -> BackgroundTaskPortError {
|
||||
BackgroundTaskPortError::Invalid(error.to_string())
|
||||
}
|
||||
@ -4,6 +4,7 @@
|
||||
//! the known-projects **registry** and the **workspace** are stored as plain
|
||||
//! JSON files in the app data directory (machine-local, outside any project).
|
||||
|
||||
mod background_task;
|
||||
mod context;
|
||||
mod embedder;
|
||||
mod live_state;
|
||||
@ -15,6 +16,7 @@ mod skill;
|
||||
mod template;
|
||||
mod vector;
|
||||
|
||||
pub use background_task::{BackgroundTaskReconcileReport, FsBackgroundTaskStore};
|
||||
pub use context::IdeaiContextStore;
|
||||
#[cfg(feature = "vector-onnx")]
|
||||
pub use embedder::OnnxEmbedder;
|
||||
|
||||
Reference in New Issue
Block a user