feat(memory): embedders vectoriels réels HTTP + ONNX derrière features (LOT C1)

Remplace les StubEmbedder pour les stratégies localServer/api/localOnnx par de
vrais moteurs, chacun derrière une feature cargo off-by-default — la posture
fondatrice « rien d'imposé, zéro dépendance » (défaut none → rappel naïf) reste
byte-for-byte inchangée.

C1a (feature vector-http, reqwest rustls optional):
- HttpEmbedder couvrant localServer (Ollama/llama.cpp) et api (OpenAI/Voyage…),
  payload OpenAI-compatible /v1/embeddings, ordre restauré par index, bearer
  token lu via env var (jamais en clair), timeout client 30s.
- detect_ollama() pour la détection de l'existant (C3).

C1b (feature vector-onnx, fastembed v5 optional):
- OnnxEmbedder en-process (e5-small, dim 384), init paresseuse + spawn_blocking,
  cache modèle sous <app_data>/embedders/onnx — aucun download au build ni au
  first-run, uniquement à la demande au 1er embed.
- Catalogue RECOMMENDED_ONNX_MODELS + ONNX_CACHE_SUBDIR + onnx_model_is_cached
  exposés (sans feature) pour la config (C2) et la popup (C3).

embedder_from_profile(profile, onnx_cache_dir) dispatche feature-gated ; sans la
feature, retombe sur StubEmbedder (Unsupported) → fallback naïf via
AdaptiveMemoryRecall. Composition root (build_memory_recall) propage le cache dir.

Tests: 10 HTTP + 6 ONNX (dont 2 #[ignore] download réel) + 26 vectoriels, verts
en défaut, --features vector-http et --features vector-onnx.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-08 19:00:40 +02:00
parent b39c11a64d
commit 32398827fb
10 changed files with 2012 additions and 50 deletions

View File

@ -26,8 +26,9 @@ use domain::ports::{
use domain::profile::{EmbedderProfile, EmbedderStrategy};
use domain::project::ProjectPath;
use infrastructure::{
embedder_from_profile, should_use_vector, AdaptiveMemoryRecall, FsMemoryStore, HashEmbedder,
NaiveMemoryRecall, StubEmbedder, VectorMemoryRecall,
embedder_from_profile, onnx_model_is_cached, should_use_vector, AdaptiveMemoryRecall,
FsMemoryStore, HashEmbedder, NaiveMemoryRecall, StubEmbedder, VectorMemoryRecall,
ONNX_CACHE_SUBDIR, RECOMMENDED_ONNX_MODELS,
};
// ---------------------------------------------------------------------------
@ -255,26 +256,48 @@ async fn stub_embedder_returns_unsupported() {
#[tokio::test]
async fn embedder_from_profile_maps_each_strategy() {
// none ⇒ no embedder (recall stays naïve).
assert!(embedder_from_profile(&EmbedderProfile::none()).is_none());
// The ONNX cache dir is irrelevant to every strategy exercised here (none of
// these short-circuiting cases loads a model), so a throwaway path is fine.
let onnx_cache = std::path::Path::new("/unused-onnx-cache");
// localOnnx / localServer / api ⇒ a StubEmbedder (Unsupported), dimension kept.
for strategy in [
EmbedderStrategy::LocalOnnx,
EmbedderStrategy::LocalServer,
EmbedderStrategy::Api,
] {
let profile =
EmbedderProfile::new("e", "E", strategy, None, None, None, 24).unwrap();
let embedder = embedder_from_profile(&profile)
.unwrap_or_else(|| panic!("{strategy:?} must yield an embedder"));
assert_eq!(embedder.dimension(), 24);
// none ⇒ no embedder (recall stays naïve).
assert!(embedder_from_profile(&EmbedderProfile::none(), onnx_cache).is_none());
// localOnnx without the `vector-onnx` feature maps to a StubEmbedder
// (Unsupported); with the feature it is the real OnnxEmbedder. Dimension is
// preserved either way (the real engine validates dimension only at embed time).
let onnx =
EmbedderProfile::new("e", "E", EmbedderStrategy::LocalOnnx, None, None, None, 24).unwrap();
let embedder =
embedder_from_profile(&onnx, onnx_cache).expect("localOnnx must yield an embedder");
assert_eq!(embedder.dimension(), 24);
#[cfg(not(feature = "vector-onnx"))]
{
let err = embedder.embed(&["t".to_string()]).await.unwrap_err();
assert!(
matches!(err, EmbedderError::Unsupported(_)),
"{strategy:?} stub must be Unsupported, got {err:?}"
"localOnnx stub must be Unsupported without vector-onnx, got {err:?}"
);
}
// localServer / api: a stub *without* the `vector-http` feature (default,
// dependency-free build), a real HttpEmbedder *with* it. Either way an embedder
// is produced with the declared dimension.
for strategy in [EmbedderStrategy::LocalServer, EmbedderStrategy::Api] {
let profile = EmbedderProfile::new("e", "E", strategy, None, None, None, 24).unwrap();
let embedder = embedder_from_profile(&profile, onnx_cache)
.unwrap_or_else(|| panic!("{strategy:?} must yield an embedder"));
assert_eq!(embedder.dimension(), 24);
#[cfg(not(feature = "vector-http"))]
{
let err = embedder.embed(&["t".to_string()]).await.unwrap_err();
assert!(
matches!(err, EmbedderError::Unsupported(_)),
"{strategy:?} stub must be Unsupported without vector-http, got {err:?}"
);
}
}
}
// ===========================================================================
@ -597,6 +620,93 @@ async fn adaptive_falls_back_to_naive_when_embedder_fails() {
assert!(!got.is_empty(), "the degraded result still returns entries");
}
// ===========================================================================
// 7. Always-available ONNX data/inspection helpers (compiled WITHOUT the
// `vector-onnx` feature — they are pure data + pure-FS, no `fastembed`).
// ===========================================================================
/// A unique, self-cleaning scratch dir under the OS temp dir (project convention;
/// see tests/project_store.rs), used to exercise `onnx_model_is_cached` on real FS.
struct TempDir(std::path::PathBuf);
impl TempDir {
fn new() -> Self {
let p = std::env::temp_dir().join(format!("idea-onnx-cached-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&p).unwrap();
Self(p)
}
fn path(&self) -> &std::path::Path {
&self.0
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
#[test]
fn recommended_onnx_models_advertise_e5_small_384() {
assert_eq!(RECOMMENDED_ONNX_MODELS.len(), 1, "exactly one curated model");
let m = RECOMMENDED_ONNX_MODELS[0];
assert_eq!(m.id, "multilingual-e5-small");
assert_eq!(m.dimension, 384);
assert!(m.recommended, "the single curated model is the recommended default");
}
#[test]
fn onnx_cache_subdir_is_stable() {
assert_eq!(ONNX_CACHE_SUBDIR, "embedders/onnx");
}
#[test]
fn onnx_model_is_cached_false_on_empty_dir() {
// A fresh, empty cache dir ⇒ nothing cached.
let tmp = TempDir::new();
assert!(
!onnx_model_is_cached(tmp.path(), "multilingual-e5-small"),
"an empty cache dir reports no cached model"
);
}
#[test]
fn onnx_model_is_cached_false_when_dir_missing() {
// A path that does not exist must report `false`, never panic.
let missing = std::path::Path::new("/idea-onnx-cache-definitely-missing-xyz");
assert!(!onnx_model_is_cached(missing, "multilingual-e5-small"));
}
#[test]
fn onnx_model_is_cached_true_when_model_subdir_present_and_nonempty() {
// The heuristic looks for a non-empty subdirectory whose (lowercased) name
// *contains* the model token (slashes/underscores normalised to `-`), mirroring
// hf-hub's `models--<org>--<name>` layout.
let tmp = TempDir::new();
let model_dir = tmp.path().join("models--Qdrant--multilingual-e5-small-onnx");
std::fs::create_dir_all(&model_dir).unwrap();
// The subdir must be non-empty to count as "cached".
std::fs::write(model_dir.join("model.onnx"), b"not really a model").unwrap();
assert!(
onnx_model_is_cached(tmp.path(), "multilingual-e5-small"),
"a non-empty matching subdir ⇒ cached"
);
}
#[test]
fn onnx_model_is_cached_false_when_matching_subdir_is_empty() {
// A matching but *empty* subdir does not count as a present cached model.
let tmp = TempDir::new();
std::fs::create_dir_all(tmp.path().join("models--Qdrant--multilingual-e5-small-onnx")).unwrap();
assert!(
!onnx_model_is_cached(tmp.path(), "multilingual-e5-small"),
"an empty matching subdir is not a cached model"
);
}
// ===========================================================================
// 8. AdaptiveMemoryRecall — the étage-1/étage-2 switch (cont.).
// ===========================================================================
#[tokio::test]
async fn adaptive_empty_memory_and_zero_budget_are_empty() {
// Liskov common contract: empty list, never an error, both paths.