From ba77cbeeb50e55cce05b5d06869be6e41da287bc Mon Sep 17 00:00:00 2001 From: mathieu Date: Fri, 24 Oct 2025 15:37:54 +1100 Subject: [PATCH 01/21] bump to 0.1.5 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46de7c6..5d9eb26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2633,7 +2633,7 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vectorlite" -version = "0.1.4" +version = "0.1.5" dependencies = [ "axum", "candle-core", diff --git a/Cargo.toml b/Cargo.toml index 34b2179..08b05c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vectorlite" -version = "0.1.4" +version = "0.1.5" edition = "2024" description = "A high-performance, in-memory vector database optimized for AI agent workloads" license = "Apache-2.0" From f3815324d876d954ea407f4dc5758765717e1638 Mon Sep 17 00:00:00 2001 From: mathieu Date: Fri, 24 Oct 2025 17:13:59 +1100 Subject: [PATCH 02/21] prevent duplication of vectors --- src/client.rs | 2 +- src/index/flat.rs | 4 +- src/index/hnsw.rs | 89 +++++--- src/index/hnsw_memory_optimized.rs | 314 +++++++++++++++++++++++++++++ src/lib.rs | 4 +- 5 files changed, 380 insertions(+), 33 deletions(-) create mode 100644 src/index/hnsw_memory_optimized.rs diff --git a/src/client.rs b/src/client.rs index 19679fb..1d8f817 100644 --- a/src/client.rs +++ b/src/client.rs @@ -380,7 +380,7 @@ impl Collection { pub fn get_vector(&self, id: u64) -> VectorLiteResult> { let index = self.index.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for get_vector".to_string()))?; - Ok(index.get_vector(id).cloned()) + Ok(index.get_vector(id)) } pub fn get_info(&self) -> VectorLiteResult { diff --git a/src/index/flat.rs b/src/index/flat.rs index 46e0be8..6d286f3 100644 --- a/src/index/flat.rs +++ b/src/index/flat.rs @@ -119,8 +119,8 @@ impl VectorIndex for FlatIndex { self.data.is_empty() } - fn get_vector(&self, id: u64) -> Option<&Vector> { - self.data.iter().find(|e| e.id == id) + fn get_vector(&self, id: u64) -> Option { + self.data.iter().find(|e| e.id == id).cloned() } fn dimension(&self) -> usize { diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index c7c4d34..48b8f8e 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -46,6 +46,13 @@ use serde::{Deserialize, Serialize, Deserializer}; use space::{Metric, Neighbor}; use hnsw::{Hnsw, Searcher}; use crate::{Vector, VectorIndex, SearchResult, SimilarityMetric}; + +// Separate structure for metadata only - no embedding values +#[derive(Debug, Clone, Serialize, Deserialize)] +struct VectorMetadata { + text: String, + metadata: Option, +} #[derive(Default, Clone)] struct Euclidean; @@ -91,8 +98,11 @@ pub struct HNSWIndex { id_to_index: HashMap, // Mapping from internal HNSW index to custom ID index_to_id: HashMap, - // Store vectors for retrieval by ID. - vectors: HashMap, + // Store only metadata (text + JSON), not the full Vector + metadata: HashMap, + // Store vector values separately for similarity calculations + // This is still more memory efficient than storing full Vector structs + vector_values: HashMap>, } impl HNSWIndex { @@ -108,13 +118,14 @@ impl HNSWIndex { dim, id_to_index: HashMap::new(), index_to_id: HashMap::new(), - vectors: HashMap::new(), + metadata: HashMap::new(), + vector_values: HashMap::new(), } } /// Get the maximum ID from the stored vectors pub fn max_id(&self) -> Option { - self.vectors.keys().max().copied() + self.metadata.keys().max().copied() } } @@ -126,7 +137,8 @@ impl<'de> Deserialize<'de> for HNSWIndex { #[derive(Deserialize)] struct Temp { dim: usize, - vectors: HashMap, + metadata: HashMap, + vector_values: HashMap>, } let data = Temp::deserialize(deserializer)?; @@ -137,14 +149,14 @@ impl<'de> Deserialize<'de> for HNSWIndex { let mut new_id_to_index = HashMap::new(); let mut new_index_to_id = HashMap::new(); - for (id, vector) in &data.vectors { - if vector.values.len() != data.dim { + for (id, values) in &data.vector_values { + if values.len() != data.dim { return Err(serde::de::Error::custom(format!( "Vector dimension mismatch: expected {}, got {}", - data.dim, vector.values.len() + data.dim, values.len() ))); } - let internal_index = hnsw.insert(vector.values.clone(), &mut searcher); + let internal_index = hnsw.insert(values.clone(), &mut searcher); new_id_to_index.insert(*id, internal_index); new_index_to_id.insert(internal_index, *id); } @@ -160,7 +172,8 @@ impl<'de> Deserialize<'de> for HNSWIndex { dim: data.dim, id_to_index: new_id_to_index, index_to_id: new_index_to_id, - vectors: data.vectors, + metadata: data.metadata, + vector_values: data.vector_values, }) } } @@ -175,11 +188,19 @@ impl VectorIndex for HNSWIndex { return Err(format!("Vector ID {} already exists", vector.id)); } + // Store the embedding in HNSW (takes ownership, no clone needed) let internal_index = self.hnsw.insert(vector.values.clone(), &mut self.searcher); + // Store metadata and values separately + let vector_metadata = VectorMetadata { + text: vector.text, + metadata: vector.metadata, + }; + self.id_to_index.insert(vector.id, internal_index); self.index_to_id.insert(internal_index, vector.id); - self.vectors.insert(vector.id, vector); + self.metadata.insert(vector.id, vector_metadata); + self.vector_values.insert(vector.id, vector.values); Ok(()) } @@ -193,7 +214,8 @@ impl VectorIndex for HNSWIndex { // Since HNSW doesn't support deletion, we just remove the reference to the node in the mapping self.id_to_index.remove(&id); self.index_to_id.remove(&internal_index); - self.vectors.remove(&id); + self.metadata.remove(&id); + self.vector_values.remove(&id); Ok(()) } @@ -203,7 +225,7 @@ impl VectorIndex for HNSWIndex { return Vec::new(); } - if self.vectors.is_empty() { + if self.metadata.is_empty() { return Vec::new(); } @@ -213,7 +235,7 @@ impl VectorIndex for HNSWIndex { // HNSW searches for k*2 candidates to improve accuracy in approximate search. // This compensates for the graph structure limitations and ensures we find // the best k results after recalculating with the requested similarity metric. - let max_candidates = std::cmp::min(k * 2, self.vectors.len()); + let max_candidates = std::cmp::min(k * 2, self.metadata.len()); if max_candidates == 0 { return Vec::new(); } @@ -233,14 +255,16 @@ impl VectorIndex for HNSWIndex { .filter(|n| n.index != !0) // Filter out invalid results .filter_map(|n| { self.index_to_id.get(&n.index).and_then(|&custom_id| { - self.vectors.get(&custom_id).map(|vector| { - let score = similarity_metric.calculate(&vector.values, query); - SearchResult { - id: custom_id, - score, - text: vector.text.clone(), - metadata: vector.metadata.clone() - } + self.metadata.get(&custom_id).and_then(|meta| { + self.vector_values.get(&custom_id).map(|values| { + let score = similarity_metric.calculate(values, query); + SearchResult { + id: custom_id, + score, + text: meta.text.clone(), + metadata: meta.metadata.clone() + } + }) }) }) }) @@ -252,13 +276,22 @@ impl VectorIndex for HNSWIndex { search_results } fn len(&self) -> usize { - self.vectors.len() + self.metadata.len() } fn is_empty(&self) -> bool { - self.vectors.is_empty() + self.metadata.is_empty() } - fn get_vector(&self, id: u64) -> Option<&Vector> { - self.vectors.get(&id) + fn get_vector(&self, id: u64) -> Option { + self.metadata.get(&id).and_then(|meta| { + self.vector_values.get(&id).map(|values| { + Vector { + id, + values: values.clone(), + text: meta.text.clone(), + metadata: meta.metadata.clone(), + } + }) + }) } fn dimension(&self) -> usize { self.dim @@ -268,8 +301,8 @@ impl VectorIndex for HNSWIndex { impl Debug for HNSWIndex { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("HNSWIndex") - .field("len", &self.vectors.len()) - .field("is_empty", &self.vectors.is_empty()) + .field("len", &self.metadata.len()) + .field("is_empty", &self.metadata.is_empty()) .field("dimension", &self.dim) .finish() } diff --git a/src/index/hnsw_memory_optimized.rs b/src/index/hnsw_memory_optimized.rs new file mode 100644 index 0000000..3d34544 --- /dev/null +++ b/src/index/hnsw_memory_optimized.rs @@ -0,0 +1,314 @@ +//! Memory-optimized HNSW implementation that eliminates triple storage +//! +//! This implementation separates the HNSW index data (embeddings) from the metadata (text + JSON), +//! reducing memory usage by ~50% while maintaining the same functionality. + +use std::collections::HashMap; +use std::fmt::{Formatter, Debug}; +use rand::rngs::StdRng; +use serde::{Deserialize, Serialize, Deserializer}; +use space::{Metric, Neighbor}; +use hnsw::{Hnsw, Searcher}; +use crate::{Vector, VectorIndex, SearchResult, SimilarityMetric}; + +// Configuration constants (same as original) +const MAXIMUM_NUMBER_CONNECTIONS: usize = if cfg!(feature = "memory-optimized") { + 8 +} else if cfg!(feature = "high-accuracy") { + 32 +} else { + 16 +}; + +const MAXIMUM_NUMBER_CONNECTIONS_0: usize = if cfg!(feature = "memory-optimized") { + 16 +} else if cfg!(feature = "high-accuracy") { + 64 +} else { + 32 +}; + +#[derive(Default, Clone)] +struct Euclidean; + +impl Metric> for Euclidean { + type Unit = u64; + + fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { + let sum_sq = a.iter() + .zip(b.iter()) + .map(|(x, y)| (x - y).powi(2)) + .sum::(); + (sum_sq.sqrt() * 1000.0) as u64 + } +} + +// Separate structure for metadata only - no embedding values +#[derive(Debug, Clone, Serialize, Deserialize)] +struct VectorMetadata { + text: String, + metadata: Option, +} + +/// Memory-optimized HNSW index that separates embeddings from metadata +/// +/// This implementation reduces memory usage by ~50% compared to the original +/// by storing only metadata separately from the HNSW index data. +#[derive(Clone, Serialize)] +pub struct HNSWIndexOptimized { + #[serde(skip)] + hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, + #[serde(skip)] + searcher: Searcher, + + dim: usize, + // Mapping from custom ID to internal HNSW index + id_to_index: HashMap, + // Mapping from internal HNSW index to custom ID + index_to_id: HashMap, + // Store only metadata (text + JSON), not the full Vector + metadata: HashMap, + // Store vector values separately for similarity calculations + // This is still more memory efficient than storing full Vector structs + vector_values: HashMap>, +} + +impl HNSWIndexOptimized { + pub fn new(dim: usize) -> Self { + if dim == 0 { + panic!("HNSW index dimension cannot be 0"); + } + let hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0> = Hnsw::new(Euclidean); + let searcher = Searcher::new(); + Self { + hnsw, + searcher, + dim, + id_to_index: HashMap::new(), + index_to_id: HashMap::new(), + metadata: HashMap::new(), + vector_values: HashMap::new(), + } + } + + /// Get the maximum ID from the stored vectors + pub fn max_id(&self) -> Option { + self.metadata.keys().max().copied() + } +} + +impl<'de> Deserialize<'de> for HNSWIndexOptimized { + fn deserialize(deserializer: D) -> Result + where D: Deserializer<'de> + { + // Use an anonymous struct to match the JSON structure + #[derive(Deserialize)] + struct Temp { + dim: usize, + metadata: HashMap, + vector_values: HashMap>, + } + + let data = Temp::deserialize(deserializer)?; + + let mut hnsw = Hnsw::new(Euclidean); + let mut searcher = Searcher::new(); + + let mut new_id_to_index = HashMap::new(); + let mut new_index_to_id = HashMap::new(); + + for (id, values) in &data.vector_values { + if values.len() != data.dim { + return Err(serde::de::Error::custom(format!( + "Vector dimension mismatch: expected {}, got {}", + data.dim, values.len() + ))); + } + let internal_index = hnsw.insert(values.clone(), &mut searcher); + new_id_to_index.insert(*id, internal_index); + new_index_to_id.insert(internal_index, *id); + } + + if data.dim == 0 { + return Err(serde::de::Error::custom("Invalid dimension: cannot be 0")); + } + + Ok(HNSWIndexOptimized { + hnsw, + searcher, + dim: data.dim, + id_to_index: new_id_to_index, + index_to_id: new_index_to_id, + metadata: data.metadata, + vector_values: data.vector_values, + }) + } +} + +impl VectorIndex for HNSWIndexOptimized { + fn add(&mut self, vector: Vector) -> Result<(), String> { + if vector.values.len() != self.dim { + return Err(format!("Vector dimension mismatch: expected {}, got {}", self.dim, vector.values.len())); + } + + if self.id_to_index.contains_key(&vector.id) { + return Err(format!("Vector ID {} already exists", vector.id)); + } + + // Store the embedding in HNSW (takes ownership, no clone needed) + let internal_index = self.hnsw.insert(vector.values.clone(), &mut self.searcher); + + // Store metadata and values separately + let vector_metadata = VectorMetadata { + text: vector.text, + metadata: vector.metadata, + }; + + self.id_to_index.insert(vector.id, internal_index); + self.index_to_id.insert(internal_index, vector.id); + self.metadata.insert(vector.id, vector_metadata); + self.vector_values.insert(vector.id, vector.values); + + Ok(()) + } + + fn delete(&mut self, id: u64) -> Result<(), String> { + if !self.id_to_index.contains_key(&id) { + return Err(format!("Vector ID {} does not exist", id)); + } + + let internal_index = self.id_to_index[&id]; + + // Since HNSW doesn't support deletion, we just remove the reference to the node in the mapping + self.id_to_index.remove(&id); + self.index_to_id.remove(&internal_index); + self.metadata.remove(&id); + self.vector_values.remove(&id); + + Ok(()) + } + + fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec { + let mut results = self.hnsw.search(query, &self.searcher, k * 2); + + // Get candidate vectors and recalculate with the requested similarity metric + let mut search_results: Vec = results.iter() + .filter(|n| n.index != !0) // Filter out invalid results + .filter_map(|n| { + self.index_to_id.get(&n.index).and_then(|&custom_id| { + self.metadata.get(&custom_id).and_then(|meta| { + self.vector_values.get(&custom_id).map(|values| { + let score = similarity_metric.calculate(values, query); + SearchResult { + id: custom_id, + score, + text: meta.text.clone(), + metadata: meta.metadata.clone() + } + }) + }) + }) + }) + .collect(); + + // Sort by similarity score and take top k + search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap()); + search_results.truncate(k); + search_results + } + + fn len(&self) -> usize { + self.metadata.len() + } + + fn is_empty(&self) -> bool { + self.metadata.is_empty() + } + + fn get_vector(&self, id: u64) -> Option { + self.metadata.get(&id).and_then(|meta| { + self.vector_values.get(&id).map(|values| { + Vector { + id, + values: values.clone(), + text: meta.text.clone(), + metadata: meta.metadata.clone(), + } + }) + }) + } + + fn dimension(&self) -> usize { + self.dim + } +} + +impl Debug for HNSWIndexOptimized { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HNSWIndexOptimized") + .field("len", &self.metadata.len()) + .field("is_empty", &self.metadata.is_empty()) + .field("dimension", &self.dim) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_create_hnswindex_optimized() { + let hnsw = HNSWIndexOptimized::new(3); + assert!(hnsw.is_empty()); + assert_eq!(hnsw.dimension(), 3); + } + + #[test] + fn test_add_vector_optimized() { + let mut hnsw = HNSWIndexOptimized::new(3); + let vector = Vector { + id: 1, + values: vec![1.0, 2.0, 3.0], + text: "test".to_string(), + metadata: None, + }; + + assert!(hnsw.add(vector).is_ok()); + assert_eq!(hnsw.len(), 1); + assert!(!hnsw.is_empty()); + } + + #[test] + fn test_memory_efficiency() { + let mut hnsw = HNSWIndexOptimized::new(3); + + // Add a vector with large text and metadata + let large_text = "x".repeat(1000); + let large_metadata = serde_json::json!({ + "content": "x".repeat(500), + "tags": vec!["tag1", "tag2", "tag3"], + "nested": { + "data": "x".repeat(200) + } + }); + + let vector = Vector { + id: 1, + values: vec![1.0, 2.0, 3.0], + text: large_text, + metadata: Some(large_metadata), + }; + + assert!(hnsw.add(vector).is_ok()); + + // Verify we can retrieve the vector + let retrieved = hnsw.get_vector(1); + assert!(retrieved.is_some()); + let retrieved = retrieved.unwrap(); + assert_eq!(retrieved.id, 1); + assert_eq!(retrieved.values, vec![1.0, 2.0, 3.0]); + assert!(retrieved.text.len() > 500); + assert!(retrieved.metadata.is_some()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 55b773b..b5718b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,7 +233,7 @@ pub trait VectorIndex { fn is_empty(&self) -> bool; /// Get a vector by its ID - fn get_vector(&self, id: u64) -> Option<&Vector>; + fn get_vector(&self, id: u64) -> Option; /// Get the dimension of vectors in this index fn dimension(&self) -> usize; @@ -306,7 +306,7 @@ impl VectorIndex for VectorIndexWrapper { } } - fn get_vector(&self, id: u64) -> Option<&Vector> { + fn get_vector(&self, id: u64) -> Option { match self { VectorIndexWrapper::Flat(index) => index.get_vector(id), VectorIndexWrapper::HNSW(index) => index.get_vector(id), From 0a95dac04fffb0955b044171f0a51b85624cf097 Mon Sep 17 00:00:00 2001 From: mathieu Date: Fri, 24 Oct 2025 17:14:34 +1100 Subject: [PATCH 03/21] Revert "bump to 0.1.5" This reverts commit ba77cbeeb50e55cce05b5d06869be6e41da287bc. --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d9eb26..46de7c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2633,7 +2633,7 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vectorlite" -version = "0.1.5" +version = "0.1.4" dependencies = [ "axum", "candle-core", diff --git a/Cargo.toml b/Cargo.toml index 08b05c6..34b2179 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vectorlite" -version = "0.1.5" +version = "0.1.4" edition = "2024" description = "A high-performance, in-memory vector database optimized for AI agent workloads" license = "Apache-2.0" From 67f8d9492de5aa66ce00cc11a863806ccab59347 Mon Sep 17 00:00:00 2001 From: mathieu Date: Fri, 24 Oct 2025 17:15:09 +1100 Subject: [PATCH 04/21] remove slop --- src/index/hnsw_memory_optimized.rs | 314 ----------------------------- 1 file changed, 314 deletions(-) delete mode 100644 src/index/hnsw_memory_optimized.rs diff --git a/src/index/hnsw_memory_optimized.rs b/src/index/hnsw_memory_optimized.rs deleted file mode 100644 index 3d34544..0000000 --- a/src/index/hnsw_memory_optimized.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Memory-optimized HNSW implementation that eliminates triple storage -//! -//! This implementation separates the HNSW index data (embeddings) from the metadata (text + JSON), -//! reducing memory usage by ~50% while maintaining the same functionality. - -use std::collections::HashMap; -use std::fmt::{Formatter, Debug}; -use rand::rngs::StdRng; -use serde::{Deserialize, Serialize, Deserializer}; -use space::{Metric, Neighbor}; -use hnsw::{Hnsw, Searcher}; -use crate::{Vector, VectorIndex, SearchResult, SimilarityMetric}; - -// Configuration constants (same as original) -const MAXIMUM_NUMBER_CONNECTIONS: usize = if cfg!(feature = "memory-optimized") { - 8 -} else if cfg!(feature = "high-accuracy") { - 32 -} else { - 16 -}; - -const MAXIMUM_NUMBER_CONNECTIONS_0: usize = if cfg!(feature = "memory-optimized") { - 16 -} else if cfg!(feature = "high-accuracy") { - 64 -} else { - 32 -}; - -#[derive(Default, Clone)] -struct Euclidean; - -impl Metric> for Euclidean { - type Unit = u64; - - fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { - let sum_sq = a.iter() - .zip(b.iter()) - .map(|(x, y)| (x - y).powi(2)) - .sum::(); - (sum_sq.sqrt() * 1000.0) as u64 - } -} - -// Separate structure for metadata only - no embedding values -#[derive(Debug, Clone, Serialize, Deserialize)] -struct VectorMetadata { - text: String, - metadata: Option, -} - -/// Memory-optimized HNSW index that separates embeddings from metadata -/// -/// This implementation reduces memory usage by ~50% compared to the original -/// by storing only metadata separately from the HNSW index data. -#[derive(Clone, Serialize)] -pub struct HNSWIndexOptimized { - #[serde(skip)] - hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, - #[serde(skip)] - searcher: Searcher, - - dim: usize, - // Mapping from custom ID to internal HNSW index - id_to_index: HashMap, - // Mapping from internal HNSW index to custom ID - index_to_id: HashMap, - // Store only metadata (text + JSON), not the full Vector - metadata: HashMap, - // Store vector values separately for similarity calculations - // This is still more memory efficient than storing full Vector structs - vector_values: HashMap>, -} - -impl HNSWIndexOptimized { - pub fn new(dim: usize) -> Self { - if dim == 0 { - panic!("HNSW index dimension cannot be 0"); - } - let hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0> = Hnsw::new(Euclidean); - let searcher = Searcher::new(); - Self { - hnsw, - searcher, - dim, - id_to_index: HashMap::new(), - index_to_id: HashMap::new(), - metadata: HashMap::new(), - vector_values: HashMap::new(), - } - } - - /// Get the maximum ID from the stored vectors - pub fn max_id(&self) -> Option { - self.metadata.keys().max().copied() - } -} - -impl<'de> Deserialize<'de> for HNSWIndexOptimized { - fn deserialize(deserializer: D) -> Result - where D: Deserializer<'de> - { - // Use an anonymous struct to match the JSON structure - #[derive(Deserialize)] - struct Temp { - dim: usize, - metadata: HashMap, - vector_values: HashMap>, - } - - let data = Temp::deserialize(deserializer)?; - - let mut hnsw = Hnsw::new(Euclidean); - let mut searcher = Searcher::new(); - - let mut new_id_to_index = HashMap::new(); - let mut new_index_to_id = HashMap::new(); - - for (id, values) in &data.vector_values { - if values.len() != data.dim { - return Err(serde::de::Error::custom(format!( - "Vector dimension mismatch: expected {}, got {}", - data.dim, values.len() - ))); - } - let internal_index = hnsw.insert(values.clone(), &mut searcher); - new_id_to_index.insert(*id, internal_index); - new_index_to_id.insert(internal_index, *id); - } - - if data.dim == 0 { - return Err(serde::de::Error::custom("Invalid dimension: cannot be 0")); - } - - Ok(HNSWIndexOptimized { - hnsw, - searcher, - dim: data.dim, - id_to_index: new_id_to_index, - index_to_id: new_index_to_id, - metadata: data.metadata, - vector_values: data.vector_values, - }) - } -} - -impl VectorIndex for HNSWIndexOptimized { - fn add(&mut self, vector: Vector) -> Result<(), String> { - if vector.values.len() != self.dim { - return Err(format!("Vector dimension mismatch: expected {}, got {}", self.dim, vector.values.len())); - } - - if self.id_to_index.contains_key(&vector.id) { - return Err(format!("Vector ID {} already exists", vector.id)); - } - - // Store the embedding in HNSW (takes ownership, no clone needed) - let internal_index = self.hnsw.insert(vector.values.clone(), &mut self.searcher); - - // Store metadata and values separately - let vector_metadata = VectorMetadata { - text: vector.text, - metadata: vector.metadata, - }; - - self.id_to_index.insert(vector.id, internal_index); - self.index_to_id.insert(internal_index, vector.id); - self.metadata.insert(vector.id, vector_metadata); - self.vector_values.insert(vector.id, vector.values); - - Ok(()) - } - - fn delete(&mut self, id: u64) -> Result<(), String> { - if !self.id_to_index.contains_key(&id) { - return Err(format!("Vector ID {} does not exist", id)); - } - - let internal_index = self.id_to_index[&id]; - - // Since HNSW doesn't support deletion, we just remove the reference to the node in the mapping - self.id_to_index.remove(&id); - self.index_to_id.remove(&internal_index); - self.metadata.remove(&id); - self.vector_values.remove(&id); - - Ok(()) - } - - fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec { - let mut results = self.hnsw.search(query, &self.searcher, k * 2); - - // Get candidate vectors and recalculate with the requested similarity metric - let mut search_results: Vec = results.iter() - .filter(|n| n.index != !0) // Filter out invalid results - .filter_map(|n| { - self.index_to_id.get(&n.index).and_then(|&custom_id| { - self.metadata.get(&custom_id).and_then(|meta| { - self.vector_values.get(&custom_id).map(|values| { - let score = similarity_metric.calculate(values, query); - SearchResult { - id: custom_id, - score, - text: meta.text.clone(), - metadata: meta.metadata.clone() - } - }) - }) - }) - }) - .collect(); - - // Sort by similarity score and take top k - search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap()); - search_results.truncate(k); - search_results - } - - fn len(&self) -> usize { - self.metadata.len() - } - - fn is_empty(&self) -> bool { - self.metadata.is_empty() - } - - fn get_vector(&self, id: u64) -> Option { - self.metadata.get(&id).and_then(|meta| { - self.vector_values.get(&id).map(|values| { - Vector { - id, - values: values.clone(), - text: meta.text.clone(), - metadata: meta.metadata.clone(), - } - }) - }) - } - - fn dimension(&self) -> usize { - self.dim - } -} - -impl Debug for HNSWIndexOptimized { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("HNSWIndexOptimized") - .field("len", &self.metadata.len()) - .field("is_empty", &self.metadata.is_empty()) - .field("dimension", &self.dim) - .finish() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_create_hnswindex_optimized() { - let hnsw = HNSWIndexOptimized::new(3); - assert!(hnsw.is_empty()); - assert_eq!(hnsw.dimension(), 3); - } - - #[test] - fn test_add_vector_optimized() { - let mut hnsw = HNSWIndexOptimized::new(3); - let vector = Vector { - id: 1, - values: vec![1.0, 2.0, 3.0], - text: "test".to_string(), - metadata: None, - }; - - assert!(hnsw.add(vector).is_ok()); - assert_eq!(hnsw.len(), 1); - assert!(!hnsw.is_empty()); - } - - #[test] - fn test_memory_efficiency() { - let mut hnsw = HNSWIndexOptimized::new(3); - - // Add a vector with large text and metadata - let large_text = "x".repeat(1000); - let large_metadata = serde_json::json!({ - "content": "x".repeat(500), - "tags": vec!["tag1", "tag2", "tag3"], - "nested": { - "data": "x".repeat(200) - } - }); - - let vector = Vector { - id: 1, - values: vec![1.0, 2.0, 3.0], - text: large_text, - metadata: Some(large_metadata), - }; - - assert!(hnsw.add(vector).is_ok()); - - // Verify we can retrieve the vector - let retrieved = hnsw.get_vector(1); - assert!(retrieved.is_some()); - let retrieved = retrieved.unwrap(); - assert_eq!(retrieved.id, 1); - assert_eq!(retrieved.values, vec![1.0, 2.0, 3.0]); - assert!(retrieved.text.len() > 500); - assert!(retrieved.metadata.is_some()); - } -} From 3c6f49212f69e824c91993b28d3276313a70abdc Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 14:51:20 +1100 Subject: [PATCH 05/21] search improvements --- src/client.rs | 9 +- src/index/hnsw.rs | 277 ++++++++++++++++++++++++++++++++++++++------- src/lib.rs | 2 +- src/persistence.rs | 2 +- 4 files changed, 245 insertions(+), 45 deletions(-) diff --git a/src/client.rs b/src/client.rs index 1d8f817..86b56a8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -81,6 +81,11 @@ impl VectorLiteClient { } pub fn create_collection(&mut self, name: &str, index_type: IndexType) -> VectorLiteResult<()> { + // Default to Euclidean metric for backward compatibility + self.create_collection_with_metric(name, index_type, SimilarityMetric::Euclidean) + } + + pub fn create_collection_with_metric(&mut self, name: &str, index_type: IndexType, metric: SimilarityMetric) -> VectorLiteResult<()> { if self.collections.contains_key(name) { return Err(VectorLiteError::CollectionAlreadyExists { name: name.to_string() }); } @@ -88,7 +93,9 @@ impl VectorLiteClient { let dimension = self.embedding_function.dimension(); let index = match index_type { IndexType::Flat => VectorIndexWrapper::Flat(crate::FlatIndex::new(dimension, Vec::new())), - IndexType::HNSW => VectorIndexWrapper::HNSW(Box::new(crate::HNSWIndex::new(dimension))), + // HNSW creates an optimized graph structure for the specified metric. + // All searches must use the same metric as the index was built with. + IndexType::HNSW => VectorIndexWrapper::HNSW(Box::new(crate::HNSWIndex::new(dimension, metric))), }; let collection = Collection { diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index 48b8f8e..2b7acf3 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -47,7 +47,7 @@ use space::{Metric, Neighbor}; use hnsw::{Hnsw, Searcher}; use crate::{Vector, VectorIndex, SearchResult, SimilarityMetric}; -// Separate structure for metadata only - no embedding values +// VectorMetadata contains the metadata for a vector without the embedding values #[derive(Debug, Clone, Serialize, Deserialize)] struct VectorMetadata { text: String, @@ -56,6 +56,15 @@ struct VectorMetadata { #[derive(Default, Clone)] struct Euclidean; +#[derive(Default, Clone)] +struct Cosine; + +#[derive(Default, Clone)] +struct Manhattan; + +#[derive(Default, Clone)] +struct DotProduct; + const MAXIMUM_NUMBER_CONNECTIONS: usize = if cfg!(feature = "memory-optimized") { 8 } else if cfg!(feature = "high-accuracy") { @@ -86,14 +95,87 @@ impl Metric> for Euclidean { } } +impl Metric> for Cosine { + type Unit = u64; + + fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { + // Cosine distance = 1 - cosine_similarity + let (dot, norm_a_sq, norm_b_sq) = a.iter() + .zip(b.iter()) + .fold((0.0, 0.0, 0.0), |(dot, a_sq, b_sq), (&x, &y)| { + (dot + x * y, a_sq + x * x, b_sq + y * y) + }); + + let norm_a = norm_a_sq.sqrt(); + let norm_b = norm_b_sq.sqrt(); + + if norm_a == 0.0 || norm_b == 0.0 { + return 1000; // Maximum distance for zero vectors + } + + let cosine_sim = dot / (norm_a * norm_b); + // Convert to distance: (1 - similarity) * 1000 + let distance = (1.0 - cosine_sim) * 1000.0; + distance as u64 + } +} + +impl Metric> for Manhattan { + type Unit = u64; + + fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { + let dist = a.iter() + .zip(b.iter()) + .map(|(&x, &y)| (x - y).abs()) + .sum::(); + (dist * 1000.0) as u64 + } +} + +impl Metric> for DotProduct { + type Unit = u64; + + fn distance(&self, a: &Vec, b: &Vec) -> Self::Unit { + // Dot product as distance (negative because higher dot product = smaller distance) + let dot = a.iter() + .zip(b.iter()) + .map(|(&x, &y)| x * y) + .sum::(); + // Convert to positive distance: 1000 - dot (clamped) + let normalized = (1000.0 - dot.max(-1000.0).min(1000.0)) as u64; + normalized + } +} + +/// Enum to hold different HNSW index types for different metrics +#[derive(Clone)] +enum HNSWIndexInternal { + Euclidean { + hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, + searcher: Searcher, + }, + Cosine { + hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, + searcher: Searcher, + }, + Manhattan { + hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, + searcher: Searcher, + }, + DotProduct { + hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, + searcher: Searcher, + }, +} + #[derive(Clone, Serialize)] pub struct HNSWIndex { #[serde(skip)] - hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0>, - #[serde(skip)] - searcher: Searcher, + index_internal: HNSWIndexInternal, dim: usize, + // The similarity metric this index was optimized for + metric: SimilarityMetric, // Mapping from custom ID to internal HNSW index id_to_index: HashMap, // Mapping from internal HNSW index to custom ID @@ -106,22 +188,55 @@ pub struct HNSWIndex { } impl HNSWIndex { - pub fn new(dim: usize) -> Self { + pub fn new(dim: usize, metric: SimilarityMetric) -> Self { if dim == 0 { panic!("HNSW index dimension cannot be 0"); } - let hnsw: Hnsw, StdRng, MAXIMUM_NUMBER_CONNECTIONS, MAXIMUM_NUMBER_CONNECTIONS_0> = Hnsw::new(Euclidean); - let searcher = Searcher::new(); + + // Create HNSW with the specific metric for its graph structure. + // This ensures the HNSW graph is optimized for the intended similarity metric. + let index_internal = match metric { + SimilarityMetric::Euclidean => { + HNSWIndexInternal::Euclidean { + hnsw: Hnsw::new(Euclidean), + searcher: Searcher::new(), + } + }, + SimilarityMetric::Cosine => { + HNSWIndexInternal::Cosine { + hnsw: Hnsw::new(Cosine), + searcher: Searcher::new(), + } + }, + SimilarityMetric::Manhattan => { + HNSWIndexInternal::Manhattan { + hnsw: Hnsw::new(Manhattan), + searcher: Searcher::new(), + } + }, + SimilarityMetric::DotProduct => { + HNSWIndexInternal::DotProduct { + hnsw: Hnsw::new(DotProduct), + searcher: Searcher::new(), + } + }, + }; + Self { - hnsw, - searcher, + index_internal, dim, + metric, id_to_index: HashMap::new(), index_to_id: HashMap::new(), metadata: HashMap::new(), vector_values: HashMap::new(), } } + + /// Get the metric this index was built for + pub fn metric(&self) -> SimilarityMetric { + self.metric + } /// Get the maximum ID from the stored vectors pub fn max_id(&self) -> Option { @@ -137,18 +252,51 @@ impl<'de> Deserialize<'de> for HNSWIndex { #[derive(Deserialize)] struct Temp { dim: usize, + #[serde(default)] // Default to Euclidean for backward compatibility + metric: SimilarityMetric, metadata: HashMap, vector_values: HashMap>, } let data = Temp::deserialize(deserializer)?; - let mut hnsw = Hnsw::new(Euclidean); - let mut searcher = Searcher::new(); + // Verify that the HNSW index was created with the correct dimension + if data.dim == 0 { + return Err(serde::de::Error::custom("Invalid dimension: cannot be 0")); + } + + // Create the appropriate HNSW index based on the metric + let mut index_internal = match data.metric { + SimilarityMetric::Euclidean => { + HNSWIndexInternal::Euclidean { + hnsw: Hnsw::new(Euclidean), + searcher: Searcher::new(), + } + }, + SimilarityMetric::Cosine => { + HNSWIndexInternal::Cosine { + hnsw: Hnsw::new(Cosine), + searcher: Searcher::new(), + } + }, + SimilarityMetric::Manhattan => { + HNSWIndexInternal::Manhattan { + hnsw: Hnsw::new(Manhattan), + searcher: Searcher::new(), + } + }, + SimilarityMetric::DotProduct => { + HNSWIndexInternal::DotProduct { + hnsw: Hnsw::new(DotProduct), + searcher: Searcher::new(), + } + }, + }; let mut new_id_to_index = HashMap::new(); let mut new_index_to_id = HashMap::new(); + // Insert all vectors into the appropriate HNSW index for (id, values) in &data.vector_values { if values.len() != data.dim { return Err(serde::de::Error::custom(format!( @@ -156,20 +304,30 @@ impl<'de> Deserialize<'de> for HNSWIndex { data.dim, values.len() ))); } - let internal_index = hnsw.insert(values.clone(), &mut searcher); + + let internal_index = match &mut index_internal { + HNSWIndexInternal::Euclidean { hnsw, searcher } => { + hnsw.insert(values.clone(), searcher) + }, + HNSWIndexInternal::Cosine { hnsw, searcher } => { + hnsw.insert(values.clone(), searcher) + }, + HNSWIndexInternal::Manhattan { hnsw, searcher } => { + hnsw.insert(values.clone(), searcher) + }, + HNSWIndexInternal::DotProduct { hnsw, searcher } => { + hnsw.insert(values.clone(), searcher) + }, + }; + new_id_to_index.insert(*id, internal_index); new_index_to_id.insert(internal_index, *id); } - // Verify that the HNSW index was created with the correct dimension - if data.dim == 0 { - return Err(serde::de::Error::custom("Invalid dimension: cannot be 0")); - } - Ok(HNSWIndex { - hnsw, - searcher, + index_internal, dim: data.dim, + metric: data.metric, id_to_index: new_id_to_index, index_to_id: new_index_to_id, metadata: data.metadata, @@ -188,8 +346,20 @@ impl VectorIndex for HNSWIndex { return Err(format!("Vector ID {} already exists", vector.id)); } - // Store the embedding in HNSW (takes ownership, no clone needed) - let internal_index = self.hnsw.insert(vector.values.clone(), &mut self.searcher); + let internal_index = match &mut self.index_internal { + HNSWIndexInternal::Euclidean { hnsw, searcher } => { + hnsw.insert(vector.values.clone(), searcher) + }, + HNSWIndexInternal::Cosine { hnsw, searcher } => { + hnsw.insert(vector.values.clone(), searcher) + }, + HNSWIndexInternal::Manhattan { hnsw, searcher } => { + hnsw.insert(vector.values.clone(), searcher) + }, + HNSWIndexInternal::DotProduct { hnsw, searcher } => { + hnsw.insert(vector.values.clone(), searcher) + }, + }; // Store metadata and values separately let vector_metadata = VectorMetadata { @@ -221,7 +391,14 @@ impl VectorIndex for HNSWIndex { } fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec { if query.len() != self.dim { - eprintln!("Warning: Query dimension mismatch. Expected {}, got {}. Returning empty results.", self.dim, query.len()); + eprintln!("Error: Query dimension mismatch. Expected {}, got {}. Returning empty results.", self.dim, query.len()); + return Vec::new(); + } + + // Reject searches that don't match the metric the index was built for + // HNSW's graph structure is optimized for a specific distance metric + if similarity_metric != self.metric { + eprintln!("Error: HNSW index was built for {:?} similarity, but search requested {:?}. Search rejected.", self.metric, similarity_metric); return Vec::new(); } @@ -230,12 +407,7 @@ impl VectorIndex for HNSWIndex { } let query_vec = query.to_vec(); - - let mut searcher: Searcher = Searcher::new(); - // HNSW searches for k*2 candidates to improve accuracy in approximate search. - // This compensates for the graph structure limitations and ensures we find - // the best k results after recalculating with the requested similarity metric. - let max_candidates = std::cmp::min(k * 2, self.metadata.len()); + let max_candidates = std::cmp::min(k, self.metadata.len()); if max_candidates == 0 { return Vec::new(); } @@ -248,15 +420,36 @@ impl VectorIndex for HNSWIndex { max_candidates ]; - let results = self.hnsw.nearest(&query_vec, max_candidates, &mut searcher, &mut neighbors); + // Use the appropriate HNSW index based on the metric + let results = match &self.index_internal { + HNSWIndexInternal::Euclidean { hnsw, .. } => { + let mut searcher: Searcher = Searcher::new(); + hnsw.nearest(&query_vec, max_candidates, &mut searcher, &mut neighbors) + }, + HNSWIndexInternal::Cosine { hnsw, .. } => { + let mut searcher: Searcher = Searcher::new(); + hnsw.nearest(&query_vec, max_candidates, &mut searcher, &mut neighbors) + }, + HNSWIndexInternal::Manhattan { hnsw, .. } => { + let mut searcher: Searcher = Searcher::new(); + hnsw.nearest(&query_vec, max_candidates, &mut searcher, &mut neighbors) + }, + HNSWIndexInternal::DotProduct { hnsw, .. } => { + let mut searcher: Searcher = Searcher::new(); + hnsw.nearest(&query_vec, max_candidates, &mut searcher, &mut neighbors) + }, + }; - // Get candidate vectors and recalculate with the requested similarity metric + // Now that HNSW uses the correct metric internally, we can directly use the results + // and convert distances to similarity scores let mut search_results: Vec = results.iter() .filter(|n| n.index != !0) // Filter out invalid results .filter_map(|n| { self.index_to_id.get(&n.index).and_then(|&custom_id| { self.metadata.get(&custom_id).and_then(|meta| { self.vector_values.get(&custom_id).map(|values| { + // Calculate similarity score from the distance + // The HNSW returns distances, we need to convert to similarity let score = similarity_metric.calculate(values, query); SearchResult { id: custom_id, @@ -309,14 +502,14 @@ impl Debug for HNSWIndex { } #[test] fn test_create_hnswindex() { - let hnsw = HNSWIndex::new(3); + let hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); assert!(hnsw.is_empty()); assert_eq!(hnsw.dimension(), 3); } #[test] fn test_add_vector() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vector = Vector { id: 1, values: vec![1.0, 2.0, 3.0], @@ -331,7 +524,7 @@ fn test_add_vector() { #[test] fn test_add_vector_dimension_mismatch() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vector = Vector { id: 1, values: vec![1.0, 2.0], // Wrong dimension @@ -345,7 +538,7 @@ fn test_add_vector_dimension_mismatch() { #[test] fn test_search_basic() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vectors = vec![ Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "test".to_string(), metadata: None }, @@ -375,7 +568,7 @@ fn test_search_basic() { #[test] fn test_search_empty_index() { - let hnsw = HNSWIndex::new(3); + let hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let query = vec![1.0, 2.0, 3.0]; let results = hnsw.search(&query, 5, SimilarityMetric::Euclidean); @@ -384,7 +577,7 @@ fn test_search_empty_index() { #[test] fn test_id_mapping() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); // Add vectors with custom IDs let vectors = vec![ @@ -416,7 +609,7 @@ fn test_id_mapping() { #[test] fn test_duplicate_id_error() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vector1 = Vector { id: 1, values: vec![1.0, 2.0, 3.0], text: "test".to_string(), metadata: None }; let vector2 = Vector { id: 1, values: vec![4.0, 5.0, 6.0], text: "test".to_string(), metadata: None }; // Same ID @@ -427,7 +620,7 @@ fn test_duplicate_id_error() { #[test] fn test_delete_vector() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vector = Vector { id: 42, values: vec![1.0, 2.0, 3.0], text: "test".to_string(), metadata: None }; assert!(hnsw.add(vector).is_ok()); @@ -446,7 +639,7 @@ fn test_delete_vector() { fn test_feature_flags() { // Test that the constants are properly set based on features // This test will only pass if the correct feature is enabled - let hnsw = HNSWIndex::new(3); + let hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); // Verify the HNSW was created successfully assert!(hnsw.is_empty()); @@ -461,7 +654,7 @@ fn test_serialization_deserialization() { use serde_json; // Create an HNSW index with some data - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); let vectors = vec![ Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "test".to_string(), metadata: None }, Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "test".to_string(), metadata: None }, @@ -534,7 +727,7 @@ fn test_empty_hnsw_serialization_deserialization() { use serde_json; // Create an empty HNSW index - let empty_hnsw = HNSWIndex::new(3); + let empty_hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); assert!(empty_hnsw.is_empty()); assert_eq!(empty_hnsw.dimension(), 3); @@ -555,7 +748,7 @@ fn test_empty_hnsw_serialization_deserialization() { #[test] fn test_search_with_limited_vectors() { - let mut hnsw = HNSWIndex::new(3); + let mut hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); // Add only 3 vectors let vectors = vec![ diff --git a/src/lib.rs b/src/lib.rs index b5718b4..a458cc4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -336,7 +336,7 @@ impl VectorIndex for VectorIndexWrapper { /// let cosine_score = SimilarityMetric::Cosine.calculate(&a, &b); /// let euclidean_score = SimilarityMetric::Euclidean.calculate(&a, &b); /// ``` -#[derive(Debug, Clone, Copy, PartialEq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize)] pub enum SimilarityMetric { /// Cosine similarity - scale-invariant, good for normalized embeddings /// Range: [-1, 1], where 1 is identical diff --git a/src/persistence.rs b/src/persistence.rs index 29a708c..0cccccc 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -272,7 +272,7 @@ mod tests { let file_path = temp_dir.path().join("test_hnsw_collection.vlc"); // Create HNSW collection - let hnsw_index = HNSWIndex::new(3); + let hnsw_index = HNSWIndex::new(3, SimilarityMetric::Euclidean); let index = VectorIndexWrapper::HNSW(Box::new(hnsw_index)); let collection = Collection::new("test_hnsw_collection".to_string(), index); From 41f667c96137fc1f5b15b7e0692183628003f681 Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 14:54:52 +1100 Subject: [PATCH 06/21] improve score calculation --- src/index/hnsw.rs | 54 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index 2b7acf3..4f4a206 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -47,6 +47,33 @@ use space::{Metric, Neighbor}; use hnsw::{Hnsw, Searcher}; use crate::{Vector, VectorIndex, SearchResult, SimilarityMetric}; +/// Convert distance to similarity score for the given metric +fn convert_distance_to_similarity(distance: f64, metric: SimilarityMetric) -> f64 { + match metric { + SimilarityMetric::Euclidean => { + // For Euclidean: similarity = 1 / (1 + distance) + 1.0 / (1.0 + distance) + }, + SimilarityMetric::Cosine => { + // For Cosine: distance = 1 - similarity, so similarity = 1 - distance + // But distance is [0, 2000] scaled, so we divide by 1000 + let cos_distance = distance / 1000.0; + 1.0 - cos_distance + }, + SimilarityMetric::Manhattan => { + // For Manhattan: similarity = 1 / (1 + distance) + 1.0 / (1.0 + distance) + }, + SimilarityMetric::DotProduct => { + // For DotProduct: distance = 1000 - dot_product (clamped) + // So: dot_product = 1000 - distance + // We want similarity to range [0, 1] where higher dot product = higher similarity + // Convert: similarity = (1000 - distance) / 1000, normalized to [0, 1] + ((1000.0 - distance) / 1000.0).max(0.0).min(1.0) + }, + } +} + // VectorMetadata contains the metadata for a vector without the embedding values #[derive(Debug, Clone, Serialize, Deserialize)] struct VectorMetadata { @@ -440,24 +467,23 @@ impl VectorIndex for HNSWIndex { }, }; - // Now that HNSW uses the correct metric internally, we can directly use the results - // and convert distances to similarity scores + // Convert HNSW distances to similarity scores + // The HNSW returns distances in its native Unit (u64 scaled by 1000) let mut search_results: Vec = results.iter() .filter(|n| n.index != !0) // Filter out invalid results .filter_map(|n| { self.index_to_id.get(&n.index).and_then(|&custom_id| { - self.metadata.get(&custom_id).and_then(|meta| { - self.vector_values.get(&custom_id).map(|values| { - // Calculate similarity score from the distance - // The HNSW returns distances, we need to convert to similarity - let score = similarity_metric.calculate(values, query); - SearchResult { - id: custom_id, - score, - text: meta.text.clone(), - metadata: meta.metadata.clone() - } - }) + self.metadata.get(&custom_id).map(|meta| { + // Convert u64 distance back to f64, then to similarity + let distance = n.distance as f64 / 1000.0; + let score = convert_distance_to_similarity(distance, similarity_metric); + + SearchResult { + id: custom_id, + score, + text: meta.text.clone(), + metadata: meta.metadata.clone() + } }) }) }) From c2e26af4ff43073613c68d4f74f6aedd0e1feb2c Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 15:01:10 +1100 Subject: [PATCH 07/21] fix tests --- src/client.rs | 12 ++++++------ src/index/hnsw.rs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client.rs b/src/client.rs index 86b56a8..3504d49 100644 --- a/src/client.rs +++ b/src/client.rs @@ -684,8 +684,8 @@ mod tests { let info = client.get_collection_info("hnsw_collection").unwrap(); assert_eq!(info.count, 2); - // Test search - let results = client.search_text_in_collection("hnsw_collection", "First", 1, SimilarityMetric::Cosine).unwrap(); + // Test search with Euclidean (must match the index metric) + let results = client.search_text_in_collection("hnsw_collection", "First", 1, SimilarityMetric::Euclidean).unwrap(); assert_eq!(results.len(), 1); } @@ -748,8 +748,8 @@ mod tests { // Create a separate embedding function for testing let test_embedding_fn = MockEmbeddingFunction::new(3); - // Test search on original collection using text search (like the working test) - let results = collection.search_text("First", 1, SimilarityMetric::Cosine, &test_embedding_fn).unwrap(); + // Test search on original collection using text search (must match index metric) + let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); assert_eq!(results.len(), 1); // Save to temporary file @@ -771,8 +771,8 @@ mod tests { assert_eq!(info.dimension, 3); assert!(!info.is_empty); - // Test search functionality using text search - let results = loaded_collection.search_text("First", 1, SimilarityMetric::Cosine, &test_embedding_fn).unwrap(); + // Test search functionality using text search (must match index metric) + let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); assert_eq!(results.len(), 1); } diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index 4f4a206..b027e40 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -30,11 +30,11 @@ //! use vectorlite::{HNSWIndex, Vector, SimilarityMetric, VectorIndex}; //! //! # fn example() -> Result<(), Box> { -//! let mut index = HNSWIndex::new(384); +//! let mut index = HNSWIndex::new(384, SimilarityMetric::Euclidean); //! let vector = Vector { id: 1, values: vec![0.1; 384], text: "test".to_string(), metadata: None }; //! //! index.add(vector)?; -//! let results = index.search(&[0.1; 384], 5, SimilarityMetric::Cosine); +//! let results = index.search(&[0.1; 384], 5, SimilarityMetric::Euclidean); //! # Ok(()) //! # } //! ``` From 7062030d929080a226333ceb103de5e300c64083 Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 15:25:31 +1100 Subject: [PATCH 08/21] fixes --- README.md | 4 +- src/client.rs | 49 ++++---- src/lib.rs | 2 +- src/server.rs | 16 ++- tests/http_integration_test.rs | 12 +- tests/metric_enforcement_test.rs | 192 +++++++++++++++++++++++++++++++ 6 files changed, 239 insertions(+), 36 deletions(-) create mode 100644 tests/metric_enforcement_test.rs diff --git a/README.md b/README.md index 52984f1..d88f97d 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ docker build \ See [Hierarchical Navigable Small World](https://arxiv.org/abs/1603.09320). +Note: HNSW indices must specify a similarity metric at creation and search with the same metric. + ### Configuration profiles for HNSW | Profile | Features | Use Case | @@ -111,7 +113,7 @@ use serde_json::json; fn main() -> Result<(), Box> { let client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); - client.create_collection("quotes", IndexType::HNSW)?; + client.create_collection_with_metric("quotes", IndexType::HNSW, SimilarityMetric::Cosine)?; let id = client.add_text_to_collection( "quotes", diff --git a/src/client.rs b/src/client.rs index 3504d49..c86f0b5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -17,7 +17,7 @@ //! let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); //! //! // Create a collection -//! client.create_collection("documents", IndexType::HNSW)?; +//! client.create_collection("documents", IndexType::HNSW, SimilarityMetric::Cosine)?; //! //! // Add text (auto-generates embedding) //! let id = client.add_text_to_collection("documents", "Hello world", None)?; @@ -54,11 +54,11 @@ use crate::errors::{VectorLiteError, VectorLiteResult}; /// # Examples /// /// ```rust -/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType}; +/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric}; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); -/// client.create_collection("docs", IndexType::HNSW)?; +/// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; /// # Ok(()) /// # } /// ``` @@ -80,12 +80,7 @@ impl VectorLiteClient { } } - pub fn create_collection(&mut self, name: &str, index_type: IndexType) -> VectorLiteResult<()> { - // Default to Euclidean metric for backward compatibility - self.create_collection_with_metric(name, index_type, SimilarityMetric::Euclidean) - } - - pub fn create_collection_with_metric(&mut self, name: &str, index_type: IndexType, metric: SimilarityMetric) -> VectorLiteResult<()> { + pub fn create_collection(&mut self, name: &str, index_type: IndexType, metric: SimilarityMetric) -> VectorLiteResult<()> { if self.collections.contains_key(name) { return Err(VectorLiteError::CollectionAlreadyExists { name: name.to_string() }); } @@ -185,16 +180,16 @@ impl VectorLiteClient { /// # Examples /// /// ```rust -/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType}; +/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric}; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); /// /// // For small datasets with exact search requirements -/// client.create_collection("small_data", IndexType::Flat)?; +/// client.create_collection("small_data", IndexType::Flat, SimilarityMetric::Cosine)?; /// /// // For large datasets with approximate search tolerance -/// client.create_collection("large_data", IndexType::HNSW)?; +/// client.create_collection("large_data", IndexType::HNSW, SimilarityMetric::Cosine)?; /// # Ok(()) /// # } /// ``` @@ -242,11 +237,11 @@ type CollectionRef = Arc; /// # Examples /// /// ```rust -/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType}; +/// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric}; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); -/// client.create_collection("docs", IndexType::HNSW)?; +/// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; /// /// let info = client.get_collection_info("docs")?; /// println!("Collection '{}' has {} vectors of dimension {}", @@ -431,12 +426,12 @@ impl Collection { /// # Examples /// /// ```rust - /// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType}; + /// use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric}; /// use std::path::Path; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); - /// client.create_collection("docs", IndexType::HNSW)?; + /// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; /// client.add_text_to_collection("docs", "Hello world", None)?; /// /// let collection = client.get_collection("docs").unwrap(); @@ -521,7 +516,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - let result = client.create_collection("test_collection", IndexType::Flat); + let result = client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean); assert!(result.is_ok()); // Check collection exists @@ -535,10 +530,10 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create first collection - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); // Try to create duplicate - let result = client.create_collection("test_collection", IndexType::Flat); + let result = client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), VectorLiteError::CollectionAlreadyExists { .. })); } @@ -549,7 +544,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); // Get collection let collection = client.get_collection("test_collection"); @@ -567,7 +562,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); assert!(client.has_collection("test_collection")); // Delete collection @@ -586,7 +581,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); // Add text let result = client.add_text_to_collection("test_collection", "Hello world", None); @@ -622,7 +617,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); // Test initial state let info = client.get_collection_info("test_collection").unwrap(); @@ -672,7 +667,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create HNSW collection - client.create_collection("hnsw_collection", IndexType::HNSW).unwrap(); + client.create_collection("hnsw_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); // Add some text let id1 = client.add_text_to_collection("hnsw_collection", "First document", None).unwrap(); @@ -697,7 +692,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection and add some data - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); client.add_text_to_collection("test_collection", "Another text", None).unwrap(); @@ -734,7 +729,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create HNSW collection and add some data - client.create_collection("test_hnsw_collection", IndexType::HNSW).unwrap(); + client.create_collection("test_hnsw_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); client.add_text_to_collection("test_hnsw_collection", "First document", None).unwrap(); client.add_text_to_collection("test_hnsw_collection", "Second document", None).unwrap(); @@ -781,7 +776,7 @@ mod tests { let embedding_fn = MockEmbeddingFunction::new(3); let mut client = VectorLiteClient::new(Box::new(embedding_fn)); - client.create_collection("test_collection", IndexType::Flat).unwrap(); + client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let collection = client.get_collection("test_collection").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index a458cc4..28062c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,7 @@ //! fn main() -> Result<(), Box> { //! let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); //! -//! client.create_collection("quotes", IndexType::HNSW)?; +//! client.create_collection("quotes", IndexType::HNSW, SimilarityMetric::Cosine)?; //! //! let id = client.add_text_to_collection( //! "quotes", diff --git a/src/server.rs b/src/server.rs index efbd121..e3e163a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -73,6 +73,8 @@ use crate::errors::{VectorLiteError, VectorLiteResult}; pub struct CreateCollectionRequest { pub name: String, pub index_type: String, // "flat" or "hnsw" + #[serde(default)] + pub metric: String, // "cosine", "euclidean", "manhattan", "dotproduct" } #[derive(Debug, Serialize)] @@ -192,8 +194,20 @@ async fn create_collection( } }; + // Parse metric, default to cosine if not provided + let metric = if payload.metric.is_empty() { + SimilarityMetric::Cosine // Default to cosine if not provided + } else { + match parse_similarity_metric(&payload.metric) { + Ok(m) => m, + Err(e) => { + return Err(e.status_code()); + } + } + }; + let mut client = state.write().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.create_collection(&payload.name, index_type) { + match client.create_collection(&payload.name, index_type, metric) { Ok(_) => { info!("Created collection: {}", payload.name); Ok(Json(CreateCollectionResponse { diff --git a/tests/http_integration_test.rs b/tests/http_integration_test.rs index 21116ec..00ca51b 100644 --- a/tests/http_integration_test.rs +++ b/tests/http_integration_test.rs @@ -133,7 +133,7 @@ async fn test_create_duplicate_collection() { #[tokio::test] async fn test_get_collection_info() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() @@ -155,7 +155,7 @@ async fn test_get_collection_info() { #[tokio::test] async fn test_add_text_to_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let payload = json!({ @@ -181,7 +181,7 @@ async fn test_add_text_to_collection() { #[tokio::test] async fn test_search_text() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -212,7 +212,7 @@ async fn test_search_text() { #[tokio::test] async fn test_get_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -234,7 +234,7 @@ async fn test_get_vector() { #[tokio::test] async fn test_delete_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -255,7 +255,7 @@ async fn test_delete_vector() { #[tokio::test] async fn test_delete_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() diff --git a/tests/metric_enforcement_test.rs b/tests/metric_enforcement_test.rs new file mode 100644 index 0000000..092a3fc --- /dev/null +++ b/tests/metric_enforcement_test.rs @@ -0,0 +1,192 @@ +use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric, HNSWIndex, VectorIndex, Vector}; + +#[test] +fn test_hnsw_metric_enforcement() { + // Test that HNSW index enforces the metric it was built with + let mut hnsw_euclidean = HNSWIndex::new(3, SimilarityMetric::Euclidean); + let mut hnsw_cosine = HNSWIndex::new(3, SimilarityMetric::Cosine); + let mut hnsw_manhattan = HNSWIndex::new(3, SimilarityMetric::Manhattan); + let mut hnsw_dotproduct = HNSWIndex::new(3, SimilarityMetric::DotProduct); + + // Add the same vectors to all indices + let vectors = vec![ + Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "first".to_string(), metadata: None }, + Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "second".to_string(), metadata: None }, + Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "third".to_string(), metadata: None }, + ]; + + for vector in &vectors { + hnsw_euclidean.add(vector.clone()).unwrap(); + hnsw_cosine.add(vector.clone()).unwrap(); + hnsw_manhattan.add(vector.clone()).unwrap(); + hnsw_dotproduct.add(vector.clone()).unwrap(); + } + + let query = vec![1.1, 0.1, 0.1]; + + // Test that each index only accepts its own metric + + // Euclidean index should only work with Euclidean metric + let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Euclidean); + assert!(!results.is_empty(), "Euclidean index should work with Euclidean metric"); + + let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Cosine); + assert!(results.is_empty(), "Euclidean index should reject Cosine metric"); + + let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Manhattan); + assert!(results.is_empty(), "Euclidean index should reject Manhattan metric"); + + let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::DotProduct); + assert!(results.is_empty(), "Euclidean index should reject DotProduct metric"); + + // Cosine index should only work with Cosine metric + let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Cosine); + assert!(!results.is_empty(), "Cosine index should work with Cosine metric"); + + let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Euclidean); + assert!(results.is_empty(), "Cosine index should reject Euclidean metric"); + + let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Manhattan); + assert!(results.is_empty(), "Cosine index should reject Manhattan metric"); + + let results = hnsw_cosine.search(&query, 2, SimilarityMetric::DotProduct); + assert!(results.is_empty(), "Cosine index should reject DotProduct metric"); + + // Manhattan index should only work with Manhattan metric + let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Manhattan); + assert!(!results.is_empty(), "Manhattan index should work with Manhattan metric"); + + let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Euclidean); + assert!(results.is_empty(), "Manhattan index should reject Euclidean metric"); + + let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Cosine); + assert!(results.is_empty(), "Manhattan index should reject Cosine metric"); + + let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::DotProduct); + assert!(results.is_empty(), "Manhattan index should reject DotProduct metric"); + + // DotProduct index should only work with DotProduct metric + let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::DotProduct); + assert!(!results.is_empty(), "DotProduct index should work with DotProduct metric"); + + let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Euclidean); + assert!(results.is_empty(), "DotProduct index should reject Euclidean metric"); + + let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Cosine); + assert!(results.is_empty(), "DotProduct index should reject Cosine metric"); + + let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Manhattan); + assert!(results.is_empty(), "DotProduct index should reject Manhattan metric"); +} + +#[test] +fn test_client_metric_enforcement() { + // Test that collections created with specific metrics enforce them + let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new().unwrap())); + + // Create collections with different metrics + client.create_collection("euclidean_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("cosine_collection", IndexType::HNSW, SimilarityMetric::Cosine).unwrap(); + client.create_collection("manhattan_collection", IndexType::HNSW, SimilarityMetric::Manhattan).unwrap(); + client.create_collection("dotproduct_collection", IndexType::HNSW, SimilarityMetric::DotProduct).unwrap(); + + // Add the same text to all collections + client.add_text_to_collection("euclidean_collection", "Hello world", None).unwrap(); + client.add_text_to_collection("cosine_collection", "Hello world", None).unwrap(); + client.add_text_to_collection("manhattan_collection", "Hello world", None).unwrap(); + client.add_text_to_collection("dotproduct_collection", "Hello world", None).unwrap(); + + // Test that searching with the correct metric works + let results = client.search_text_in_collection("euclidean_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + assert!(!results.is_empty(), "Euclidean collection should work with Euclidean metric"); + + let results = client.search_text_in_collection("cosine_collection", "hello", 1, SimilarityMetric::Cosine).unwrap(); + assert!(!results.is_empty(), "Cosine collection should work with Cosine metric"); + + let results = client.search_text_in_collection("manhattan_collection", "hello", 1, SimilarityMetric::Manhattan).unwrap(); + assert!(!results.is_empty(), "Manhattan collection should work with Manhattan metric"); + + let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, SimilarityMetric::DotProduct).unwrap(); + assert!(!results.is_empty(), "DotProduct collection should work with DotProduct metric"); + + // Test that searching with wrong metrics returns empty results (due to HNSW rejection) + let results = client.search_text_in_collection("euclidean_collection", "hello", 1, SimilarityMetric::Cosine).unwrap(); + assert!(results.is_empty(), "Euclidean collection should reject Cosine metric"); + + let results = client.search_text_in_collection("cosine_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + assert!(results.is_empty(), "Cosine collection should reject Euclidean metric"); + + let results = client.search_text_in_collection("manhattan_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + assert!(results.is_empty(), "Manhattan collection should reject Euclidean metric"); + + let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + assert!(results.is_empty(), "DotProduct collection should reject Euclidean metric"); +} + +#[test] +fn test_hnsw_metric_accuracy() { + // Test that each metric type returns correct results for its specific distance calculation + + let mut hnsw_euclidean = HNSWIndex::new(3, SimilarityMetric::Euclidean); + let mut hnsw_cosine = HNSWIndex::new(3, SimilarityMetric::Cosine); + + // Add orthogonal vectors (for cosine, these have 0 similarity) + let vectors = vec![ + Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "x-axis".to_string(), metadata: None }, + Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "y-axis".to_string(), metadata: None }, + Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "z-axis".to_string(), metadata: None }, + Vector { id: 4, values: vec![1.0, 1.0, 1.0], text: "diagonal".to_string(), metadata: None }, + ]; + + for vector in &vectors { + hnsw_euclidean.add(vector.clone()).unwrap(); + hnsw_cosine.add(vector.clone()).unwrap(); + } + + // Query close to x-axis + let query = vec![0.9, 0.1, 0.1]; + + // For Euclidean, closest should be x-axis (id=1) + let euclidean_results = hnsw_euclidean.search(&query, 1, SimilarityMetric::Euclidean); + assert_eq!(euclidean_results.len(), 1); + assert_eq!(euclidean_results[0].id, 1, "Euclidean should find x-axis as closest"); + + // For Cosine, the angle matters more than distance + // The normalized query direction is very close to x-axis + let cosine_results = hnsw_cosine.search(&query, 1, SimilarityMetric::Cosine); + assert_eq!(cosine_results.len(), 1); + assert_eq!(cosine_results[0].id, 1, "Cosine should also find x-axis as most similar"); +} + +#[test] +fn test_flat_index_accepts_any_metric() { + // Flat index should accept any metric at search time since it's a brute-force search + use vectorlite::FlatIndex; + + let mut flat = FlatIndex::new(3, Vec::new()); + + let vectors = vec![ + Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "first".to_string(), metadata: None }, + Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "second".to_string(), metadata: None }, + Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "third".to_string(), metadata: None }, + ]; + + for vector in &vectors { + flat.add(vector.clone()).unwrap(); + } + + let query = vec![1.1, 0.1, 0.1]; + + // Flat index should work with all metrics + let results = flat.search(&query, 2, SimilarityMetric::Euclidean); + assert!(!results.is_empty(), "Flat index should work with Euclidean"); + + let results = flat.search(&query, 2, SimilarityMetric::Cosine); + assert!(!results.is_empty(), "Flat index should work with Cosine"); + + let results = flat.search(&query, 2, SimilarityMetric::Manhattan); + assert!(!results.is_empty(), "Flat index should work with Manhattan"); + + let results = flat.search(&query, 2, SimilarityMetric::DotProduct); + assert!(!results.is_empty(), "Flat index should work with DotProduct"); +} From 6f5ee044a3f02b0d04c74b87c390ee73394f516e Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 15:39:26 +1100 Subject: [PATCH 09/21] fix external api --- src/client.rs | 81 ++++++++++++++++++++------------ src/lib.rs | 27 ++++++++++- src/server.rs | 10 ++-- tests/http_integration_test.rs | 12 ++--- tests/metric_enforcement_test.rs | 24 +++++----- 5 files changed, 100 insertions(+), 54 deletions(-) diff --git a/src/client.rs b/src/client.rs index c86f0b5..7f6a0d5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -17,7 +17,7 @@ //! let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); //! //! // Create a collection -//! client.create_collection("documents", IndexType::HNSW, SimilarityMetric::Cosine)?; +//! client.create_collection("documents", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; //! //! // Add text (auto-generates embedding) //! let id = client.add_text_to_collection("documents", "Hello world", None)?; @@ -27,7 +27,7 @@ //! "documents", //! "hello", //! 5, -//! SimilarityMetric::Cosine +//! None // Auto-detects from HNSW index metric //! )?; //! # Ok(()) //! # } @@ -58,7 +58,7 @@ use crate::errors::{VectorLiteError, VectorLiteResult}; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); -/// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; +/// client.create_collection("docs", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; /// # Ok(()) /// # } /// ``` @@ -80,17 +80,22 @@ impl VectorLiteClient { } } - pub fn create_collection(&mut self, name: &str, index_type: IndexType, metric: SimilarityMetric) -> VectorLiteResult<()> { + pub fn create_collection(&mut self, name: &str, index_type: IndexType, metric: Option) -> VectorLiteResult<()> { if self.collections.contains_key(name) { return Err(VectorLiteError::CollectionAlreadyExists { name: name.to_string() }); } let dimension = self.embedding_function.dimension(); let index = match index_type { - IndexType::Flat => VectorIndexWrapper::Flat(crate::FlatIndex::new(dimension, Vec::new())), - // HNSW creates an optimized graph structure for the specified metric. - // All searches must use the same metric as the index was built with. - IndexType::HNSW => VectorIndexWrapper::HNSW(Box::new(crate::HNSWIndex::new(dimension, metric))), + IndexType::Flat => { + // Flat index supports all metrics, so we don't need to store one + VectorIndexWrapper::Flat(crate::FlatIndex::new(dimension, Vec::new())) + }, + IndexType::HNSW => { + // HNSW requires a metric to build the graph structure + let used_metric = metric.unwrap_or(SimilarityMetric::Cosine); // Default to Cosine + VectorIndexWrapper::HNSW(Box::new(crate::HNSWIndex::new(dimension, used_metric))) + }, }; let collection = Collection { @@ -131,11 +136,28 @@ impl VectorLiteClient { } - pub fn search_text_in_collection(&self, collection_name: &str, query_text: &str, k: usize, similarity_metric: SimilarityMetric) -> VectorLiteResult> { + pub fn search_text_in_collection(&self, collection_name: &str, query_text: &str, k: usize, similarity_metric: Option) -> VectorLiteResult> { let collection = self.collections.get(collection_name) .ok_or_else(|| VectorLiteError::CollectionNotFound { name: collection_name.to_string() })?; - collection.search_text(query_text, k, similarity_metric, self.embedding_function.as_ref()) + // If no metric provided, try to get it from the index (HNSW) + let metric = match similarity_metric { + Some(m) => m, + None => { + // Try to get metric from the index itself + let index_guard = collection.index.read().map_err(|_| { + VectorLiteError::LockError("Failed to acquire read lock for metric detection".to_string()) + })?; + + // Check if this is an HNSW index with a specific metric + match index_guard.metric() { + Some(m) => m, + None => SimilarityMetric::Cosine, // Default for Flat index + } + } + }; + + collection.search_text(query_text, k, metric, self.embedding_function.as_ref()) } @@ -186,10 +208,11 @@ impl VectorLiteClient { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); /// /// // For small datasets with exact search requirements -/// client.create_collection("small_data", IndexType::Flat, SimilarityMetric::Cosine)?; +/// // Flat index - metric is optional +/// client.create_collection("small_data", IndexType::Flat, None)?; /// /// // For large datasets with approximate search tolerance -/// client.create_collection("large_data", IndexType::HNSW, SimilarityMetric::Cosine)?; +/// client.create_collection("large_data", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; /// # Ok(()) /// # } /// ``` @@ -241,7 +264,7 @@ type CollectionRef = Arc; /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); -/// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; +/// client.create_collection("docs", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; /// /// let info = client.get_collection_info("docs")?; /// println!("Collection '{}' has {} vectors of dimension {}", @@ -431,7 +454,7 @@ impl Collection { /// /// # fn example() -> Result<(), Box> { /// let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); - /// client.create_collection("docs", IndexType::HNSW, SimilarityMetric::Cosine)?; + /// client.create_collection("docs", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; /// client.add_text_to_collection("docs", "Hello world", None)?; /// /// let collection = client.get_collection("docs").unwrap(); @@ -516,7 +539,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - let result = client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean); + let result = client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)); assert!(result.is_ok()); // Check collection exists @@ -530,10 +553,10 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create first collection - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); // Try to create duplicate - let result = client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean); + let result = client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), VectorLiteError::CollectionAlreadyExists { .. })); } @@ -544,7 +567,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); // Get collection let collection = client.get_collection("test_collection"); @@ -562,7 +585,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); assert!(client.has_collection("test_collection")); // Delete collection @@ -581,7 +604,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); // Add text let result = client.add_text_to_collection("test_collection", "Hello world", None); @@ -617,7 +640,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); // Test initial state let info = client.get_collection_info("test_collection").unwrap(); @@ -641,7 +664,7 @@ mod tests { assert_eq!(info.count, 2); // Test search - let results = client.search_text_in_collection("test_collection", "Hello", 1, SimilarityMetric::Cosine).unwrap(); + let results = client.search_text_in_collection("test_collection", "Hello", 1, Some(SimilarityMetric::Cosine)).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].id, 0); @@ -667,7 +690,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create HNSW collection - client.create_collection("hnsw_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("hnsw_collection", IndexType::HNSW, Some(SimilarityMetric::Euclidean)).unwrap(); // Add some text let id1 = client.add_text_to_collection("hnsw_collection", "First document", None).unwrap(); @@ -680,7 +703,7 @@ mod tests { assert_eq!(info.count, 2); // Test search with Euclidean (must match the index metric) - let results = client.search_text_in_collection("hnsw_collection", "First", 1, SimilarityMetric::Euclidean).unwrap(); + let results = client.search_text_in_collection("hnsw_collection", "First", 1, Some(SimilarityMetric::Euclidean)).unwrap(); assert_eq!(results.len(), 1); } @@ -692,7 +715,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection and add some data - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); client.add_text_to_collection("test_collection", "Another text", None).unwrap(); @@ -729,7 +752,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create HNSW collection and add some data - client.create_collection("test_hnsw_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_hnsw_collection", IndexType::HNSW, Some(SimilarityMetric::Euclidean)).unwrap(); client.add_text_to_collection("test_hnsw_collection", "First document", None).unwrap(); client.add_text_to_collection("test_hnsw_collection", "Second document", None).unwrap(); @@ -744,7 +767,7 @@ mod tests { let test_embedding_fn = MockEmbeddingFunction::new(3); // Test search on original collection using text search (must match index metric) - let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); + let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API assert_eq!(results.len(), 1); // Save to temporary file @@ -767,7 +790,7 @@ mod tests { assert!(!info.is_empty); // Test search functionality using text search (must match index metric) - let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); + let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API assert_eq!(results.len(), 1); } @@ -776,7 +799,7 @@ mod tests { let embedding_fn = MockEmbeddingFunction::new(3); let mut client = VectorLiteClient::new(Box::new(embedding_fn)); - client.create_collection("test_collection", IndexType::Flat, SimilarityMetric::Euclidean).unwrap(); + client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let collection = client.get_collection("test_collection").unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 28062c9..4740c58 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,8 @@ //! fn main() -> Result<(), Box> { //! let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); //! -//! client.create_collection("quotes", IndexType::HNSW, SimilarityMetric::Cosine)?; +//! // Create HNSW collection with specific metric +//! client.create_collection("quotes", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; //! //! let id = client.add_text_to_collection( //! "quotes", @@ -57,11 +58,12 @@ //! })) //! )?; //! +//! // Search without specifying metric - automatically uses the index's metric //! let results = client.search_text_in_collection( //! "quotes", //! "beach games", //! 3, -//! SimilarityMetric::Cosine, +//! None, // Auto-detects from HNSW index //! )?; //! //! for result in &results { @@ -78,11 +80,13 @@ //! - **Complexity**: O(n) search, O(1) insert //! - **Memory**: Linear with dataset size //! - **Use Case**: Small datasets (< 10K vectors) or exact search requirements +//! - **Metric Flexibility**: Supports all similarity metrics dynamically //! //! ### HNSWIndex //! - **Complexity**: O(log n) search, O(log n) insert //! - **Memory**: ~2-3x vector size due to graph structure //! - **Use Case**: Large datasets with approximate search tolerance +//! - **Metric Flexibility**: Built for a specific metric; searches automatically use the index's metric //! //! ## Similarity Metrics //! @@ -321,6 +325,25 @@ impl VectorIndex for VectorIndexWrapper { } } +impl VectorIndexWrapper { + /// Get the similarity metric this index was built for (HNSW only) + /// Returns None for Flat indexes (which support all metrics) + pub fn metric(&self) -> Option { + match self { + VectorIndexWrapper::Flat(_) => None, + VectorIndexWrapper::HNSW(index) => Some(index.metric()), + } + } + + /// Get the index type + pub fn index_type(&self) -> IndexType { + match self { + VectorIndexWrapper::Flat(_) => IndexType::Flat, + VectorIndexWrapper::HNSW(_) => IndexType::HNSW, + } + } +} + /// Similarity metrics for vector comparison /// /// Different metrics are suitable for different use cases and vector characteristics. diff --git a/src/server.rs b/src/server.rs index e3e163a..f2f3d08 100644 --- a/src/server.rs +++ b/src/server.rs @@ -194,12 +194,12 @@ async fn create_collection( } }; - // Parse metric, default to cosine if not provided + // Parse metric - optional for Flat index, required for HNSW let metric = if payload.metric.is_empty() { - SimilarityMetric::Cosine // Default to cosine if not provided + None // No metric specified } else { match parse_similarity_metric(&payload.metric) { - Ok(m) => m, + Ok(m) => Some(m), Err(e) => { return Err(e.status_code()); } @@ -279,12 +279,12 @@ async fn search_text( let k = payload.k.unwrap_or(10); let similarity_metric = match payload.similarity_metric { Some(metric) => match parse_similarity_metric(&metric) { - Ok(m) => m, + Ok(m) => Some(m), Err(e) => { return Err(e.status_code()); } }, - None => SimilarityMetric::Cosine, + None => None, // No metric specified - will auto-detect }; let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; diff --git a/tests/http_integration_test.rs b/tests/http_integration_test.rs index 00ca51b..c2e8f09 100644 --- a/tests/http_integration_test.rs +++ b/tests/http_integration_test.rs @@ -133,7 +133,7 @@ async fn test_create_duplicate_collection() { #[tokio::test] async fn test_get_collection_info() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() @@ -155,7 +155,7 @@ async fn test_get_collection_info() { #[tokio::test] async fn test_add_text_to_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let payload = json!({ @@ -181,7 +181,7 @@ async fn test_add_text_to_collection() { #[tokio::test] async fn test_search_text() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -212,7 +212,7 @@ async fn test_search_text() { #[tokio::test] async fn test_get_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -234,7 +234,7 @@ async fn test_get_vector() { #[tokio::test] async fn test_delete_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -255,7 +255,7 @@ async fn test_delete_vector() { #[tokio::test] async fn test_delete_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, vectorlite::SimilarityMetric::Cosine).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() diff --git a/tests/metric_enforcement_test.rs b/tests/metric_enforcement_test.rs index 092a3fc..05cea4b 100644 --- a/tests/metric_enforcement_test.rs +++ b/tests/metric_enforcement_test.rs @@ -85,10 +85,10 @@ fn test_client_metric_enforcement() { let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new().unwrap())); // Create collections with different metrics - client.create_collection("euclidean_collection", IndexType::HNSW, SimilarityMetric::Euclidean).unwrap(); - client.create_collection("cosine_collection", IndexType::HNSW, SimilarityMetric::Cosine).unwrap(); - client.create_collection("manhattan_collection", IndexType::HNSW, SimilarityMetric::Manhattan).unwrap(); - client.create_collection("dotproduct_collection", IndexType::HNSW, SimilarityMetric::DotProduct).unwrap(); + client.create_collection("euclidean_collection", IndexType::HNSW, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("cosine_collection", IndexType::HNSW, Some(SimilarityMetric::Cosine)).unwrap(); + client.create_collection("manhattan_collection", IndexType::HNSW, Some(SimilarityMetric::Manhattan)).unwrap(); + client.create_collection("dotproduct_collection", IndexType::HNSW, Some(SimilarityMetric::DotProduct)).unwrap(); // Add the same text to all collections client.add_text_to_collection("euclidean_collection", "Hello world", None).unwrap(); @@ -97,29 +97,29 @@ fn test_client_metric_enforcement() { client.add_text_to_collection("dotproduct_collection", "Hello world", None).unwrap(); // Test that searching with the correct metric works - let results = client.search_text_in_collection("euclidean_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + let results = client.search_text_in_collection("euclidean_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); assert!(!results.is_empty(), "Euclidean collection should work with Euclidean metric"); - let results = client.search_text_in_collection("cosine_collection", "hello", 1, SimilarityMetric::Cosine).unwrap(); + let results = client.search_text_in_collection("cosine_collection", "hello", 1, Some(SimilarityMetric::Cosine)).unwrap(); assert!(!results.is_empty(), "Cosine collection should work with Cosine metric"); - let results = client.search_text_in_collection("manhattan_collection", "hello", 1, SimilarityMetric::Manhattan).unwrap(); + let results = client.search_text_in_collection("manhattan_collection", "hello", 1, Some(SimilarityMetric::Manhattan)).unwrap(); assert!(!results.is_empty(), "Manhattan collection should work with Manhattan metric"); - let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, SimilarityMetric::DotProduct).unwrap(); + let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, Some(SimilarityMetric::DotProduct)).unwrap(); assert!(!results.is_empty(), "DotProduct collection should work with DotProduct metric"); // Test that searching with wrong metrics returns empty results (due to HNSW rejection) - let results = client.search_text_in_collection("euclidean_collection", "hello", 1, SimilarityMetric::Cosine).unwrap(); + let results = client.search_text_in_collection("euclidean_collection", "hello", 1, Some(SimilarityMetric::Cosine)).unwrap(); assert!(results.is_empty(), "Euclidean collection should reject Cosine metric"); - let results = client.search_text_in_collection("cosine_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + let results = client.search_text_in_collection("cosine_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); assert!(results.is_empty(), "Cosine collection should reject Euclidean metric"); - let results = client.search_text_in_collection("manhattan_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + let results = client.search_text_in_collection("manhattan_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); assert!(results.is_empty(), "Manhattan collection should reject Euclidean metric"); - let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, SimilarityMetric::Euclidean).unwrap(); + let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); assert!(results.is_empty(), "DotProduct collection should reject Euclidean metric"); } From 38841e23f1f2b2a196d2d1044f78cef56a7edd97 Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 15:40:57 +1100 Subject: [PATCH 10/21] readme --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d88f97d..7696d53 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,10 @@ docker build \ | --------------------- | ----------------------------------------- | ------------------------------------------------------------------ | | **Health** | `GET /health` | – | | **List collections** | `GET /collections` | – | -| **Create collection** | `POST /collections` | `{"name": "docs", "index_type": "hnsw"}` | +| **Create collection** | `POST /collections` | `{"name": "docs", "index_type": "hnsw", "metric": "cosine"}` (metric optional) | | **Delete collection** | `DELETE /collections/{name}` | – | | **Add text** | `POST /collections/{name}/text` | `{"text": "Hello world", "metadata": {...}}`| -| **Search (text)** | `POST /collections/{name}/search/text` | `{"query": "hello", "k": 5}` | +| **Search (text)** | `POST /collections/{name}/search/text` | `{"query": "hello", "k": 5}` (metric auto-detected for HNSW) | | **Get vector** | `GET /collections/{name}/vectors/{id}` | – | | **Delete vector** | `DELETE /collections/{name}/vectors/{id}` | – | | **Save collection** | `POST /collections/{name}/save` | `{"file_path": "./collection.vlc"}` | @@ -83,7 +83,7 @@ docker build \ See [Hierarchical Navigable Small World](https://arxiv.org/abs/1603.09320). -Note: HNSW indices must specify a similarity metric at creation and search with the same metric. +Note: Flat indices support all metrics dynamically. HNSW indices default to Cosine if not specified. ### Configuration profiles for HNSW @@ -111,9 +111,10 @@ use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetr use serde_json::json; fn main() -> Result<(), Box> { - let client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); + let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); - client.create_collection_with_metric("quotes", IndexType::HNSW, SimilarityMetric::Cosine)?; + // Metric optional - defaults to Cosine for HNSW + client.create_collection("quotes", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; let id = client.add_text_to_collection( "quotes", @@ -125,11 +126,12 @@ fn main() -> Result<(), Box> { })) )?; + // Metric optional - auto-detected from HNSW index let results = client.search_text_in_collection( "quotes", "beach games", 3, - SimilarityMetric::Cosine, + None, )?; for result in &results { From 63d9c212d49178cfc83e3ea2cad6145453f16901 Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 15:53:07 +1100 Subject: [PATCH 11/21] remove local tests --- README.md | 4 +- src/index/hnsw.rs | 3 +- tests/metric_enforcement_test.rs | 192 ------------------------------- 3 files changed, 3 insertions(+), 196 deletions(-) delete mode 100644 tests/metric_enforcement_test.rs diff --git a/README.md b/README.md index 7696d53..5370a48 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,10 @@ docker build \ | --------------------- | ----------------------------------------- | ------------------------------------------------------------------ | | **Health** | `GET /health` | – | | **List collections** | `GET /collections` | – | -| **Create collection** | `POST /collections` | `{"name": "docs", "index_type": "hnsw", "metric": "cosine"}` (metric optional) | +| **Create collection** | `POST /collections` | `{"name": "docs", "index_type": "hnsw", "metric": "cosine"}`| | **Delete collection** | `DELETE /collections/{name}` | – | | **Add text** | `POST /collections/{name}/text` | `{"text": "Hello world", "metadata": {...}}`| -| **Search (text)** | `POST /collections/{name}/search/text` | `{"query": "hello", "k": 5}` (metric auto-detected for HNSW) | +| **Search (text)** | `POST /collections/{name}/search/text` | `{"query": "hello", "k": 5}` | | **Get vector** | `GET /collections/{name}/vectors/{id}` | – | | **Delete vector** | `DELETE /collections/{name}/vectors/{id}` | – | | **Save collection** | `POST /collections/{name}/save` | `{"file_path": "./collection.vlc"}` | diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index b027e40..463a3af 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -209,8 +209,7 @@ pub struct HNSWIndex { index_to_id: HashMap, // Store only metadata (text + JSON), not the full Vector metadata: HashMap, - // Store vector values separately for similarity calculations - // This is still more memory efficient than storing full Vector structs + // Store vector values separately vector_values: HashMap>, } diff --git a/tests/metric_enforcement_test.rs b/tests/metric_enforcement_test.rs deleted file mode 100644 index 05cea4b..0000000 --- a/tests/metric_enforcement_test.rs +++ /dev/null @@ -1,192 +0,0 @@ -use vectorlite::{VectorLiteClient, EmbeddingGenerator, IndexType, SimilarityMetric, HNSWIndex, VectorIndex, Vector}; - -#[test] -fn test_hnsw_metric_enforcement() { - // Test that HNSW index enforces the metric it was built with - let mut hnsw_euclidean = HNSWIndex::new(3, SimilarityMetric::Euclidean); - let mut hnsw_cosine = HNSWIndex::new(3, SimilarityMetric::Cosine); - let mut hnsw_manhattan = HNSWIndex::new(3, SimilarityMetric::Manhattan); - let mut hnsw_dotproduct = HNSWIndex::new(3, SimilarityMetric::DotProduct); - - // Add the same vectors to all indices - let vectors = vec![ - Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "first".to_string(), metadata: None }, - Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "second".to_string(), metadata: None }, - Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "third".to_string(), metadata: None }, - ]; - - for vector in &vectors { - hnsw_euclidean.add(vector.clone()).unwrap(); - hnsw_cosine.add(vector.clone()).unwrap(); - hnsw_manhattan.add(vector.clone()).unwrap(); - hnsw_dotproduct.add(vector.clone()).unwrap(); - } - - let query = vec![1.1, 0.1, 0.1]; - - // Test that each index only accepts its own metric - - // Euclidean index should only work with Euclidean metric - let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Euclidean); - assert!(!results.is_empty(), "Euclidean index should work with Euclidean metric"); - - let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Cosine); - assert!(results.is_empty(), "Euclidean index should reject Cosine metric"); - - let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::Manhattan); - assert!(results.is_empty(), "Euclidean index should reject Manhattan metric"); - - let results = hnsw_euclidean.search(&query, 2, SimilarityMetric::DotProduct); - assert!(results.is_empty(), "Euclidean index should reject DotProduct metric"); - - // Cosine index should only work with Cosine metric - let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Cosine); - assert!(!results.is_empty(), "Cosine index should work with Cosine metric"); - - let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Euclidean); - assert!(results.is_empty(), "Cosine index should reject Euclidean metric"); - - let results = hnsw_cosine.search(&query, 2, SimilarityMetric::Manhattan); - assert!(results.is_empty(), "Cosine index should reject Manhattan metric"); - - let results = hnsw_cosine.search(&query, 2, SimilarityMetric::DotProduct); - assert!(results.is_empty(), "Cosine index should reject DotProduct metric"); - - // Manhattan index should only work with Manhattan metric - let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Manhattan); - assert!(!results.is_empty(), "Manhattan index should work with Manhattan metric"); - - let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Euclidean); - assert!(results.is_empty(), "Manhattan index should reject Euclidean metric"); - - let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::Cosine); - assert!(results.is_empty(), "Manhattan index should reject Cosine metric"); - - let results = hnsw_manhattan.search(&query, 2, SimilarityMetric::DotProduct); - assert!(results.is_empty(), "Manhattan index should reject DotProduct metric"); - - // DotProduct index should only work with DotProduct metric - let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::DotProduct); - assert!(!results.is_empty(), "DotProduct index should work with DotProduct metric"); - - let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Euclidean); - assert!(results.is_empty(), "DotProduct index should reject Euclidean metric"); - - let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Cosine); - assert!(results.is_empty(), "DotProduct index should reject Cosine metric"); - - let results = hnsw_dotproduct.search(&query, 2, SimilarityMetric::Manhattan); - assert!(results.is_empty(), "DotProduct index should reject Manhattan metric"); -} - -#[test] -fn test_client_metric_enforcement() { - // Test that collections created with specific metrics enforce them - let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new().unwrap())); - - // Create collections with different metrics - client.create_collection("euclidean_collection", IndexType::HNSW, Some(SimilarityMetric::Euclidean)).unwrap(); - client.create_collection("cosine_collection", IndexType::HNSW, Some(SimilarityMetric::Cosine)).unwrap(); - client.create_collection("manhattan_collection", IndexType::HNSW, Some(SimilarityMetric::Manhattan)).unwrap(); - client.create_collection("dotproduct_collection", IndexType::HNSW, Some(SimilarityMetric::DotProduct)).unwrap(); - - // Add the same text to all collections - client.add_text_to_collection("euclidean_collection", "Hello world", None).unwrap(); - client.add_text_to_collection("cosine_collection", "Hello world", None).unwrap(); - client.add_text_to_collection("manhattan_collection", "Hello world", None).unwrap(); - client.add_text_to_collection("dotproduct_collection", "Hello world", None).unwrap(); - - // Test that searching with the correct metric works - let results = client.search_text_in_collection("euclidean_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); - assert!(!results.is_empty(), "Euclidean collection should work with Euclidean metric"); - - let results = client.search_text_in_collection("cosine_collection", "hello", 1, Some(SimilarityMetric::Cosine)).unwrap(); - assert!(!results.is_empty(), "Cosine collection should work with Cosine metric"); - - let results = client.search_text_in_collection("manhattan_collection", "hello", 1, Some(SimilarityMetric::Manhattan)).unwrap(); - assert!(!results.is_empty(), "Manhattan collection should work with Manhattan metric"); - - let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, Some(SimilarityMetric::DotProduct)).unwrap(); - assert!(!results.is_empty(), "DotProduct collection should work with DotProduct metric"); - - // Test that searching with wrong metrics returns empty results (due to HNSW rejection) - let results = client.search_text_in_collection("euclidean_collection", "hello", 1, Some(SimilarityMetric::Cosine)).unwrap(); - assert!(results.is_empty(), "Euclidean collection should reject Cosine metric"); - - let results = client.search_text_in_collection("cosine_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); - assert!(results.is_empty(), "Cosine collection should reject Euclidean metric"); - - let results = client.search_text_in_collection("manhattan_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); - assert!(results.is_empty(), "Manhattan collection should reject Euclidean metric"); - - let results = client.search_text_in_collection("dotproduct_collection", "hello", 1, Some(SimilarityMetric::Euclidean)).unwrap(); - assert!(results.is_empty(), "DotProduct collection should reject Euclidean metric"); -} - -#[test] -fn test_hnsw_metric_accuracy() { - // Test that each metric type returns correct results for its specific distance calculation - - let mut hnsw_euclidean = HNSWIndex::new(3, SimilarityMetric::Euclidean); - let mut hnsw_cosine = HNSWIndex::new(3, SimilarityMetric::Cosine); - - // Add orthogonal vectors (for cosine, these have 0 similarity) - let vectors = vec![ - Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "x-axis".to_string(), metadata: None }, - Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "y-axis".to_string(), metadata: None }, - Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "z-axis".to_string(), metadata: None }, - Vector { id: 4, values: vec![1.0, 1.0, 1.0], text: "diagonal".to_string(), metadata: None }, - ]; - - for vector in &vectors { - hnsw_euclidean.add(vector.clone()).unwrap(); - hnsw_cosine.add(vector.clone()).unwrap(); - } - - // Query close to x-axis - let query = vec![0.9, 0.1, 0.1]; - - // For Euclidean, closest should be x-axis (id=1) - let euclidean_results = hnsw_euclidean.search(&query, 1, SimilarityMetric::Euclidean); - assert_eq!(euclidean_results.len(), 1); - assert_eq!(euclidean_results[0].id, 1, "Euclidean should find x-axis as closest"); - - // For Cosine, the angle matters more than distance - // The normalized query direction is very close to x-axis - let cosine_results = hnsw_cosine.search(&query, 1, SimilarityMetric::Cosine); - assert_eq!(cosine_results.len(), 1); - assert_eq!(cosine_results[0].id, 1, "Cosine should also find x-axis as most similar"); -} - -#[test] -fn test_flat_index_accepts_any_metric() { - // Flat index should accept any metric at search time since it's a brute-force search - use vectorlite::FlatIndex; - - let mut flat = FlatIndex::new(3, Vec::new()); - - let vectors = vec![ - Vector { id: 1, values: vec![1.0, 0.0, 0.0], text: "first".to_string(), metadata: None }, - Vector { id: 2, values: vec![0.0, 1.0, 0.0], text: "second".to_string(), metadata: None }, - Vector { id: 3, values: vec![0.0, 0.0, 1.0], text: "third".to_string(), metadata: None }, - ]; - - for vector in &vectors { - flat.add(vector.clone()).unwrap(); - } - - let query = vec![1.1, 0.1, 0.1]; - - // Flat index should work with all metrics - let results = flat.search(&query, 2, SimilarityMetric::Euclidean); - assert!(!results.is_empty(), "Flat index should work with Euclidean"); - - let results = flat.search(&query, 2, SimilarityMetric::Cosine); - assert!(!results.is_empty(), "Flat index should work with Cosine"); - - let results = flat.search(&query, 2, SimilarityMetric::Manhattan); - assert!(!results.is_empty(), "Flat index should work with Manhattan"); - - let results = flat.search(&query, 2, SimilarityMetric::DotProduct); - assert!(!results.is_empty(), "Flat index should work with DotProduct"); -} From fa6004922251b4275de469da06c0034082969c5b Mon Sep 17 00:00:00 2001 From: Mathieu Mailhos Date: Mon, 27 Oct 2025 16:17:49 +1100 Subject: [PATCH 12/21] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5370a48..5d9988e 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,6 @@ use serde_json::json; fn main() -> Result<(), Box> { let mut client = VectorLiteClient::new(Box::new(EmbeddingGenerator::new()?)); - // Metric optional - defaults to Cosine for HNSW client.create_collection("quotes", IndexType::HNSW, Some(SimilarityMetric::Cosine))?; let id = client.add_text_to_collection( From b9703e79ae9f36c5c83d2efe4d0a6f02455fb4d2 Mon Sep 17 00:00:00 2001 From: Mathieu Mailhos Date: Mon, 27 Oct 2025 16:18:27 +1100 Subject: [PATCH 13/21] Update src/client.rs --- src/client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 7f6a0d5..deafab8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -88,7 +88,6 @@ impl VectorLiteClient { let dimension = self.embedding_function.dimension(); let index = match index_type { IndexType::Flat => { - // Flat index supports all metrics, so we don't need to store one VectorIndexWrapper::Flat(crate::FlatIndex::new(dimension, Vec::new())) }, IndexType::HNSW => { From 5d3c35fd82489996ea89bde94f2e79c7f593e0be Mon Sep 17 00:00:00 2001 From: Mathieu Mailhos Date: Mon, 27 Oct 2025 16:21:37 +1100 Subject: [PATCH 14/21] Update src/index/hnsw.rs --- src/index/hnsw.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index 463a3af..dc1b597 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -286,7 +286,6 @@ impl<'de> Deserialize<'de> for HNSWIndex { let data = Temp::deserialize(deserializer)?; - // Verify that the HNSW index was created with the correct dimension if data.dim == 0 { return Err(serde::de::Error::custom("Invalid dimension: cannot be 0")); } From 171df207541accb167517e80ac20d90dc084caa5 Mon Sep 17 00:00:00 2001 From: mathieu Date: Mon, 27 Oct 2025 16:29:43 +1100 Subject: [PATCH 15/21] fixcomments --- src/client.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index deafab8..23ff171 100644 --- a/src/client.rs +++ b/src/client.rs @@ -139,19 +139,16 @@ impl VectorLiteClient { let collection = self.collections.get(collection_name) .ok_or_else(|| VectorLiteError::CollectionNotFound { name: collection_name.to_string() })?; - // If no metric provided, try to get it from the index (HNSW) let metric = match similarity_metric { Some(m) => m, None => { - // Try to get metric from the index itself let index_guard = collection.index.read().map_err(|_| { VectorLiteError::LockError("Failed to acquire read lock for metric detection".to_string()) })?; - // Check if this is an HNSW index with a specific metric match index_guard.metric() { Some(m) => m, - None => SimilarityMetric::Cosine, // Default for Flat index + None => SimilarityMetric::Cosine, } } }; @@ -765,7 +762,7 @@ mod tests { // Create a separate embedding function for testing let test_embedding_fn = MockEmbeddingFunction::new(3); - // Test search on original collection using text search (must match index metric) + // Test search on original collection using text search let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API assert_eq!(results.len(), 1); @@ -788,7 +785,7 @@ mod tests { assert_eq!(info.dimension, 3); assert!(!info.is_empty); - // Test search functionality using text search (must match index metric) + // Test search functionality using text search let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API assert_eq!(results.len(), 1); } From a021cdaf6b128825b18dcdba64fd653ad25c4e49 Mon Sep 17 00:00:00 2001 From: Mathieu Mailhos Date: Mon, 27 Oct 2025 16:31:52 +1100 Subject: [PATCH 16/21] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d9988e..667c721 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ docker build \ See [Hierarchical Navigable Small World](https://arxiv.org/abs/1603.09320). -Note: Flat indices support all metrics dynamically. HNSW indices default to Cosine if not specified. +Note: Flat indices support all metrics dynamically. HNSW index must be created with a default distance metric (`cosine`, `euclidean`, `manhattan` or `dotproduct`). ### Configuration profiles for HNSW From 32cfb015ce6d4fda1bee72c3b6a1e13e43278fd9 Mon Sep 17 00:00:00 2001 From: mathieu Date: Tue, 28 Oct 2025 08:23:17 +1100 Subject: [PATCH 17/21] minor changes --- src/client.rs | 19 ++++++++++--------- tests/http_integration_test.rs | 12 ++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/client.rs b/src/client.rs index 23ff171..0731a84 100644 --- a/src/client.rs +++ b/src/client.rs @@ -535,7 +535,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - let result = client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)); + let result = client.create_collection("test_collection", IndexType::Flat, None); assert!(result.is_ok()); // Check collection exists @@ -549,10 +549,10 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create first collection - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); // Try to create duplicate - let result = client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)); + let result = client.create_collection("test_collection", IndexType::Flat, None); assert!(result.is_err()); assert!(matches!(result.unwrap_err(), VectorLiteError::CollectionAlreadyExists { .. })); } @@ -563,7 +563,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); // Get collection let collection = client.get_collection("test_collection"); @@ -581,7 +581,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); assert!(client.has_collection("test_collection")); // Delete collection @@ -600,7 +600,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); // Add text let result = client.add_text_to_collection("test_collection", "Hello world", None); @@ -636,7 +636,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); // Test initial state let info = client.get_collection_info("test_collection").unwrap(); @@ -711,7 +711,7 @@ mod tests { let mut client = VectorLiteClient::new(Box::new(embedding_fn)); // Create collection and add some data - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); client.add_text_to_collection("test_collection", "Another text", None).unwrap(); @@ -795,7 +795,8 @@ mod tests { let embedding_fn = MockEmbeddingFunction::new(3); let mut client = VectorLiteClient::new(Box::new(embedding_fn)); - client.create_collection("test_collection", IndexType::Flat, Some(SimilarityMetric::Euclidean)).unwrap(); + // Flat indexes don't need a metric parameter + client.create_collection("test_collection", IndexType::Flat, None).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let collection = client.get_collection("test_collection").unwrap(); diff --git a/tests/http_integration_test.rs b/tests/http_integration_test.rs index c2e8f09..9e5026c 100644 --- a/tests/http_integration_test.rs +++ b/tests/http_integration_test.rs @@ -133,7 +133,7 @@ async fn test_create_duplicate_collection() { #[tokio::test] async fn test_get_collection_info() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() @@ -155,7 +155,7 @@ async fn test_get_collection_info() { #[tokio::test] async fn test_add_text_to_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let payload = json!({ @@ -181,7 +181,7 @@ async fn test_add_text_to_collection() { #[tokio::test] async fn test_search_text() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -212,7 +212,7 @@ async fn test_search_text() { #[tokio::test] async fn test_get_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -234,7 +234,7 @@ async fn test_get_vector() { #[tokio::test] async fn test_delete_vector() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); client.add_text_to_collection("test_collection", "Hello world", None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); @@ -255,7 +255,7 @@ async fn test_delete_vector() { #[tokio::test] async fn test_delete_collection() { let mut client = create_test_client(); - client.create_collection("test_collection", vectorlite::IndexType::Flat, Some(vectorlite::SimilarityMetric::Cosine)).unwrap(); + client.create_collection("test_collection", vectorlite::IndexType::Flat, None).unwrap(); let app = create_app(std::sync::Arc::new(std::sync::RwLock::new(client))); let request = Request::builder() From 4eff89a9f3addea4b67fc578ae8173c1b09e58e0 Mon Sep 17 00:00:00 2001 From: mathieu Date: Tue, 28 Oct 2025 08:35:56 +1100 Subject: [PATCH 18/21] add similarity unit testing --- src/index/hnsw.rs | 227 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index dc1b597..f2e773e 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -802,3 +802,230 @@ fn test_search_with_limited_vectors() { } } +/// Tests for distance to similarity conversion functions +#[cfg(test)] +mod conversion_tests { + use super::{convert_distance_to_similarity, SimilarityMetric}; + + #[test] + fn test_euclidean_distance_conversion() { + // Test zero distance (identical vectors) + let distance = 0.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Euclidean); + assert_eq!(similarity, 1.0, "Zero distance should give similarity of 1.0"); + + // Test small distance + let distance = 0.5; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Euclidean); + let expected = 1.0 / (1.0 + 0.5); + assert!((similarity - expected).abs() < 1e-10, "Small distance conversion should be correct"); + + // Test medium distance + let distance = 1.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Euclidean); + let expected = 1.0 / (1.0 + 1.0); + assert!((similarity - expected).abs() < 1e-10, "Medium distance conversion should be correct"); + + // Test large distance + let distance = 10.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Euclidean); + let expected = 1.0 / (1.0 + 10.0); + assert!((similarity - expected).abs() < 1e-10, "Large distance conversion should be correct"); + + // Test very large distance + let distance = 100.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Euclidean); + assert!(similarity > 0.0 && similarity < 0.01, "Very large distance should give very low similarity"); + } + + #[test] + fn test_cosine_distance_conversion() { + // Test zero distance (identical vectors) + let distance = 0.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Cosine); + assert_eq!(similarity, 1.0, "Zero cosine distance should give similarity of 1.0"); + + // Test small distance (similar vectors) + let distance = 100.0; // cosine distance of 0.1 in scaled units + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Cosine); + let expected = 1.0 - (100.0 / 1000.0); + assert!((similarity - expected).abs() < 1e-10, "Small cosine distance conversion should be correct"); + + // Test medium distance + let distance = 500.0; // cosine distance of 0.5 in scaled units + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Cosine); + let expected = 1.0 - (500.0 / 1000.0); + assert!((similarity - expected).abs() < 1e-10, "Medium cosine distance conversion should be correct"); + + // Test maximum distance (opposite vectors) + let distance = 2000.0; // cosine distance of 2.0 in scaled units + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Cosine); + let expected = 1.0 - (2000.0 / 1000.0); + assert!((similarity - expected).abs() < 1e-10, "Maximum cosine distance conversion should be correct"); + + // Verify similarity is bounded + assert!(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity should be bounded"); + } + + #[test] + fn test_manhattan_distance_conversion() { + // Test zero distance (identical vectors) + let distance = 0.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Manhattan); + assert_eq!(similarity, 1.0, "Zero Manhattan distance should give similarity of 1.0"); + + // Test small distance + let distance = 1.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Manhattan); + let expected = 1.0 / (1.0 + 1.0); + assert!((similarity - expected).abs() < 1e-10, "Small Manhattan distance conversion should be correct"); + + // Test medium distance + let distance = 5.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Manhattan); + let expected = 1.0 / (1.0 + 5.0); + assert!((similarity - expected).abs() < 1e-10, "Medium Manhattan distance conversion should be correct"); + + // Test large distance + let distance = 20.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::Manhattan); + let expected = 1.0 / (1.0 + 20.0); + assert!((similarity - expected).abs() < 1e-10, "Large Manhattan distance conversion should be correct"); + } + + #[test] + fn test_dotproduct_distance_conversion() { + // Test zero distance (maximum dot product) + let distance = 0.0; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); + assert_eq!(similarity, 1.0, "Zero dot product distance should give similarity of 1.0"); + + // Test small distance + let distance = 100.0_f64; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); + let ratio: f64 = (1000.0 - 100.0) / 1000.0; + let expected: f64 = ratio.max(0.0).min(1.0); + assert!((similarity - expected).abs() < 1e-10, "Small dot product distance conversion should be correct"); + + // Test medium distance + let distance = 500.0_f64; + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); + let ratio: f64 = (1000.0 - 500.0) / 1000.0; + let expected: f64 = ratio.max(0.0).min(1.0); + assert!((similarity - expected).abs() < 1e-10, "Medium dot product distance conversion should be correct"); + + // Test maximum distance + let distance = 2000.0_f64; // negative dot product clamped + let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); + assert_eq!(similarity, 0.0, "Maximum dot product distance should give similarity of 0.0"); + + // Test that similarity is bounded [0, 1] + let distances = vec![0.0_f64, 100.0, 500.0, 1000.0, 1500.0, 2000.0]; + for dist in distances { + let sim = convert_distance_to_similarity(dist, SimilarityMetric::DotProduct); + assert!(sim >= 0.0 && sim <= 1.0, "DotProduct similarity should be in [0, 1]"); + } + } + + #[test] + fn test_conversion_monotonicity() { + // Test that similarity decreases monotonically with increasing distance + // for all metrics + + let distances = vec![0.0, 0.5, 1.0, 2.0, 5.0, 10.0]; + + for metric in &[ + SimilarityMetric::Euclidean, + SimilarityMetric::Cosine, + SimilarityMetric::Manhattan, + ] { + let mut prev_sim = 1.0; + for &dist in &distances { + let sim = convert_distance_to_similarity(dist, *metric); + assert!(sim <= prev_sim, + "Similarity should decrease as distance increases for {:?}", + metric); + prev_sim = sim; + } + } + } + + #[test] + fn test_conversion_edge_cases() { + // Test with extremely small distances + for metric in &[ + SimilarityMetric::Euclidean, + SimilarityMetric::Cosine, + SimilarityMetric::Manhattan, + SimilarityMetric::DotProduct, + ] { + let sim = convert_distance_to_similarity(0.0001, *metric); + assert!(sim > 0.9, "Extremely small distance should give high similarity"); + assert!(sim <= 1.0, "Similarity should not exceed 1.0"); + } + + // Test with extremely large distances + for metric in &[ + SimilarityMetric::Euclidean, + SimilarityMetric::Manhattan, + ] { + let sim = convert_distance_to_similarity(100000.0, *metric); + assert!(sim > 0.0, "Even very large distance should give non-zero similarity"); + assert!(sim < 0.01, "Very large distance should give very low similarity"); + } + } + + #[test] + fn test_conversion_known_vectors() { + // Test with actual vector calculations + + // Two identical vectors should have high similarity + let identical_distance_euclidean = 0.0; + let identical_distance_cosine = 0.0; + + let euclidean_sim = convert_distance_to_similarity(identical_distance_euclidean, SimilarityMetric::Euclidean); + let cosine_sim = convert_distance_to_similarity(identical_distance_cosine, SimilarityMetric::Cosine); + + assert_eq!(euclidean_sim, 1.0); + assert_eq!(cosine_sim, 1.0); + + // Opposite vectors (for cosine): [1,0,0] and [-1,0,0] + // Cosine distance = 2 (in raw form) = 2000 (scaled by 1000) + let opposite_distance = 2000.0; + let cosine_sim = convert_distance_to_similarity(opposite_distance, SimilarityMetric::Cosine); + assert!((cosine_sim - (-1.0)).abs() < 0.01, "Opposite vectors should have negative cosine similarity"); + + // Perpendicular vectors: [1,0] and [0,1] + // Cosine distance ≈ 1 = 1000 (scaled) + let perpendicular_distance = 1000.0; + let cosine_sim = convert_distance_to_similarity(perpendicular_distance, SimilarityMetric::Cosine); + assert!((cosine_sim - 0.0).abs() < 0.01, "Perpendicular vectors should have cosine similarity ≈ 0"); + } + + #[test] + fn test_scaling_factor_documentation() { + // Verify the scaling factors used in the conversions + // This helps document the behavior for future reference + + // Cosine: scaled by 1000.0 + // Maximum cosine distance is 2.0 (raw) = 2000 (scaled) + let max_cosine_distance = 2000.0; + let min_cosine_sim = convert_distance_to_similarity(max_cosine_distance, SimilarityMetric::Cosine); + assert_eq!(min_cosine_sim, -1.0, "Maximum cosine distance should yield similarity of -1.0"); + + // DotProduct: scaled by 1000.0 + // Maximum distance is when dot product is at minimum (negative) + let max_dotproduct_distance = 2000.0; + let min_dotproduct_sim = convert_distance_to_similarity(max_dotproduct_distance, SimilarityMetric::DotProduct); + assert_eq!(min_dotproduct_sim, 0.0, "Maximum dot product distance should yield similarity of 0.0"); + + // Euclidean and Manhattan use 1/(1+distance) formula + // They don't have a hard upper bound but should approach 0 + let large_distance = 1000.0; + let large_euclidean_sim = convert_distance_to_similarity(large_distance, SimilarityMetric::Euclidean); + let large_manhattan_sim = convert_distance_to_similarity(large_distance, SimilarityMetric::Manhattan); + assert!(large_euclidean_sim < 0.01 && large_euclidean_sim > 0.0); + assert!(large_manhattan_sim < 0.01 && large_manhattan_sim > 0.0); + } +} + From 4b7e1e1f277a79c3637a529cb41714d1e5cda0b6 Mon Sep 17 00:00:00 2001 From: mathieu Date: Tue, 28 Oct 2025 09:15:23 +1100 Subject: [PATCH 19/21] fix clippy --- src/client.rs | 24 +++++++++++++++---- src/embeddings.rs | 2 +- src/errors.rs | 10 ++++++++ src/index/flat.rs | 25 +++++++++++++------- src/index/hnsw.rs | 58 ++++++++++++++++++++++++---------------------- src/lib.rs | 9 +++---- src/persistence.rs | 4 ++-- 7 files changed, 84 insertions(+), 48 deletions(-) diff --git a/src/client.rs b/src/client.rs index 0731a84..6364fa1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -92,7 +92,8 @@ impl VectorLiteClient { }, IndexType::HNSW => { // HNSW requires a metric to build the graph structure - let used_metric = metric.unwrap_or(SimilarityMetric::Cosine); // Default to Cosine + // No default is provided to force explicit specification + let used_metric = metric.ok_or(VectorLiteError::MetricRequired)?; VectorIndexWrapper::HNSW(Box::new(crate::HNSWIndex::new(dimension, used_metric))) }, }; @@ -395,7 +396,8 @@ impl Collection { // Acquire read lock for search let index = self.index.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for search_text".to_string()))?; - Ok(index.search(&query_embedding, k, similarity_metric)) + + index.search(&query_embedding, k, similarity_metric) } @@ -703,7 +705,21 @@ mod tests { assert_eq!(results.len(), 1); } + #[test] + fn test_hnsw_requires_metric() { + let embedding_fn = MockEmbeddingFunction::new(3); + let mut client = VectorLiteClient::new(Box::new(embedding_fn)); + // Creating HNSW without metric should fail + let result = client.create_collection("hnsw_collection", IndexType::HNSW, None); + assert!(result.is_err()); + match result { + Err(VectorLiteError::MetricRequired) => { + // Expected error + } + _ => panic!("Expected MetricRequired error when creating HNSW without metric"), + } + } #[test] fn test_collection_save_and_load() { @@ -763,7 +779,7 @@ mod tests { let test_embedding_fn = MockEmbeddingFunction::new(3); // Test search on original collection using text search - let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API + let results = collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); assert_eq!(results.len(), 1); // Save to temporary file @@ -786,7 +802,7 @@ mod tests { assert!(!info.is_empty); // Test search functionality using text search - let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); // SimilarityMetric kept for backward compatibility in Collection API + let results = loaded_collection.search_text("First", 1, SimilarityMetric::Euclidean, &test_embedding_fn).unwrap(); assert_eq!(results.len(), 1); } diff --git a/src/embeddings.rs b/src/embeddings.rs index ecae792..a59f162 100644 --- a/src/embeddings.rs +++ b/src/embeddings.rs @@ -400,7 +400,7 @@ mod tests { #[test] fn test_batch_embedding_generation() { let generator = create_test_generator(); - let texts = vec![ + let texts = [ "first text".to_string(), "second text".to_string(), "third text".to_string(), diff --git a/src/errors.rs b/src/errors.rs index 30aac6b..51289f2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -37,6 +37,14 @@ pub enum VectorLiteError { #[error("Invalid similarity metric: {metric}. Must be 'cosine', 'euclidean', 'manhattan', or 'dotproduct'")] InvalidSimilarityMetric { metric: String }, + /// Metric mismatch between search request and index configuration + #[error("Metric mismatch: search requested {requested:?} but index was built for {index:?}")] + MetricMismatch { requested: crate::SimilarityMetric, index: crate::SimilarityMetric }, + + /// Metric required for HNSW index but not provided + #[error("HNSW index requires an explicit similarity metric. Metric must be specified when creating HNSW collections.")] + MetricRequired, + /// Embedding generation error #[error("Embedding generation failed: {0}")] EmbeddingError(#[from] crate::embeddings::EmbeddingError), @@ -70,6 +78,8 @@ impl VectorLiteError { VectorLiteError::CollectionAlreadyExists { .. } => StatusCode::CONFLICT, VectorLiteError::InvalidIndexType { .. } => StatusCode::BAD_REQUEST, VectorLiteError::InvalidSimilarityMetric { .. } => StatusCode::BAD_REQUEST, + VectorLiteError::MetricMismatch { .. } => StatusCode::BAD_REQUEST, + VectorLiteError::MetricRequired => StatusCode::BAD_REQUEST, VectorLiteError::EmbeddingError(_) => StatusCode::INTERNAL_SERVER_ERROR, VectorLiteError::PersistenceError(_) => StatusCode::INTERNAL_SERVER_ERROR, VectorLiteError::LockError(_) => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/src/index/flat.rs b/src/index/flat.rs index 6d286f3..5bb1200 100644 --- a/src/index/flat.rs +++ b/src/index/flat.rs @@ -95,7 +95,14 @@ impl VectorIndex for FlatIndex { Ok(()) } - fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec { + fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Result, crate::errors::VectorLiteError> { + if !self.data.is_empty() && query.len() != self.dim { + return Err(crate::errors::VectorLiteError::DimensionMismatch { + expected: self.dim, + actual: query.len() + }); + } + let mut similarities: Vec<_> = self.data .iter() .map(|e| SearchResult { @@ -108,7 +115,7 @@ impl VectorIndex for FlatIndex { similarities.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap()); similarities.truncate(k); - similarities + Ok(similarities) } fn len(&self) -> usize { @@ -162,7 +169,7 @@ mod tests { // Verify search works on the deserialized index let query = vec![1.1, 0.1, 0.1]; - let results = deserialized.search(&query, 2, SimilarityMetric::Cosine); + let results = deserialized.search(&query, 2, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 2); // Results should be sorted by score (highest first) @@ -186,7 +193,7 @@ mod tests { let index = FlatIndex::new(3, vectors); let query = vec![1.0, 0.0, 0.0]; - let results = index.search(&query, 2, SimilarityMetric::Cosine); + let results = index.search(&query, 2, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 2); assert_eq!(results[0].id, 1); // Most similar (identical) @@ -203,7 +210,7 @@ mod tests { let index = FlatIndex::new(2, vectors); let query = vec![0.0, 0.0]; - let results = index.search(&query, 2, SimilarityMetric::Euclidean); + let results = index.search(&query, 2, SimilarityMetric::Euclidean).unwrap(); assert_eq!(results.len(), 2); assert_eq!(results[0].id, 1); // Most similar (identical) @@ -220,7 +227,7 @@ mod tests { let index = FlatIndex::new(2, vectors); let query = vec![0.0, 0.0]; - let results = index.search(&query, 2, SimilarityMetric::Manhattan); + let results = index.search(&query, 2, SimilarityMetric::Manhattan).unwrap(); assert_eq!(results.len(), 2); assert_eq!(results[0].id, 1); // Most similar (identical) @@ -237,7 +244,7 @@ mod tests { let index = FlatIndex::new(2, vectors); let query = vec![1.0, 2.0]; - let results = index.search(&query, 2, SimilarityMetric::DotProduct); + let results = index.search(&query, 2, SimilarityMetric::DotProduct).unwrap(); assert_eq!(results.len(), 2); assert_eq!(results[0].id, 1); // Most similar (identical) @@ -255,11 +262,11 @@ mod tests { let query = vec![1.0, 2.0]; // Test with cosine similarity - let results_cosine = index.search(&query, 1, SimilarityMetric::Cosine); + let results_cosine = index.search(&query, 1, SimilarityMetric::Cosine).unwrap(); assert_eq!(results_cosine[0].id, 1); // Test with dot product - let results_dot = index.search(&query, 1, SimilarityMetric::DotProduct); + let results_dot = index.search(&query, 1, SimilarityMetric::DotProduct).unwrap(); assert_eq!(results_dot[0].id, 1); // Scores should be different diff --git a/src/index/hnsw.rs b/src/index/hnsw.rs index f2e773e..3c90b56 100644 --- a/src/index/hnsw.rs +++ b/src/index/hnsw.rs @@ -69,7 +69,7 @@ fn convert_distance_to_similarity(distance: f64, metric: SimilarityMetric) -> f6 // So: dot_product = 1000 - distance // We want similarity to range [0, 1] where higher dot product = higher similarity // Convert: similarity = (1000 - distance) / 1000, normalized to [0, 1] - ((1000.0 - distance) / 1000.0).max(0.0).min(1.0) + ((1000.0 - distance) / 1000.0).clamp(0.0, 1.0) }, } } @@ -169,8 +169,7 @@ impl Metric> for DotProduct { .map(|(&x, &y)| x * y) .sum::(); // Convert to positive distance: 1000 - dot (clamped) - let normalized = (1000.0 - dot.max(-1000.0).min(1000.0)) as u64; - normalized + (1000.0 - dot.clamp(-1000.0, 1000.0)) as u64 } } @@ -278,7 +277,6 @@ impl<'de> Deserialize<'de> for HNSWIndex { #[derive(Deserialize)] struct Temp { dim: usize, - #[serde(default)] // Default to Euclidean for backward compatibility metric: SimilarityMetric, metadata: HashMap, vector_values: HashMap>, @@ -414,27 +412,31 @@ impl VectorIndex for HNSWIndex { Ok(()) } - fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec { + fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Result, crate::errors::VectorLiteError> { if query.len() != self.dim { - eprintln!("Error: Query dimension mismatch. Expected {}, got {}. Returning empty results.", self.dim, query.len()); - return Vec::new(); + return Err(crate::errors::VectorLiteError::DimensionMismatch { + expected: self.dim, + actual: query.len() + }); } // Reject searches that don't match the metric the index was built for // HNSW's graph structure is optimized for a specific distance metric if similarity_metric != self.metric { - eprintln!("Error: HNSW index was built for {:?} similarity, but search requested {:?}. Search rejected.", self.metric, similarity_metric); - return Vec::new(); + return Err(crate::errors::VectorLiteError::MetricMismatch { + requested: similarity_metric, + index: self.metric + }); } if self.metadata.is_empty() { - return Vec::new(); + return Ok(Vec::new()); } let query_vec = query.to_vec(); let max_candidates = std::cmp::min(k, self.metadata.len()); if max_candidates == 0 { - return Vec::new(); + return Ok(Vec::new()); } let mut neighbors = vec![ @@ -490,7 +492,7 @@ impl VectorIndex for HNSWIndex { // Sort by similarity score and take top k search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap()); search_results.truncate(k); - search_results + Ok(search_results) } fn len(&self) -> usize { self.metadata.len() @@ -579,7 +581,7 @@ fn test_search_basic() { // Search for vector similar to [1.0, 0.0, 0.0] let query = vec![1.1, 0.1, 0.1]; - let results = hnsw.search(&query, 2, SimilarityMetric::Euclidean); + let results = hnsw.search(&query, 2, SimilarityMetric::Euclidean).unwrap(); assert!(!results.is_empty()); assert!(results.len() <= 2); @@ -590,14 +592,14 @@ fn test_search_basic() { } } -#[test] -fn test_search_empty_index() { - let hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); - let query = vec![1.0, 2.0, 3.0]; - let results = hnsw.search(&query, 5, SimilarityMetric::Euclidean); - - assert!(results.is_empty()); -} + #[test] + fn test_search_empty_index() { + let hnsw = HNSWIndex::new(3, SimilarityMetric::Euclidean); + let query = vec![1.0, 2.0, 3.0]; + let results = hnsw.search(&query, 5, SimilarityMetric::Euclidean).unwrap(); + + assert!(results.is_empty()); + } #[test] fn test_id_mapping() { @@ -624,7 +626,7 @@ fn test_id_mapping() { // Test that search returns the correct custom IDs let query = vec![1.1, 0.1, 0.1]; - let results = hnsw.search(&query, 2, SimilarityMetric::Euclidean); + let results = hnsw.search(&query, 2, SimilarityMetric::Euclidean).unwrap(); assert!(!results.is_empty()); // The first result should be the vector with ID 100 (most similar to [1.0, 0.0, 0.0]) @@ -719,7 +721,7 @@ fn test_serialization_deserialization() { let query = vec![1.1, 0.1, 0.1]; - let results = deserialized.search(&query, 2, SimilarityMetric::Euclidean); + let results = deserialized.search(&query, 2, SimilarityMetric::Euclidean).unwrap(); assert!(!results.is_empty(), "Search should return some results"); assert!(results.len() <= 2, "Should return at most 2 results as requested"); @@ -790,7 +792,7 @@ fn test_search_with_limited_vectors() { // Test searching for k=4 (more than we have vectors) // This should not panic and should return at most 3 results let query = vec![1.1, 0.1, 0.1]; - let results = hnsw.search(&query, 4, SimilarityMetric::Euclidean); + let results = hnsw.search(&query, 4, SimilarityMetric::Euclidean).unwrap(); // Should return at most 3 results (all available vectors) assert!(results.len() <= 3); @@ -864,7 +866,7 @@ mod conversion_tests { assert!((similarity - expected).abs() < 1e-10, "Maximum cosine distance conversion should be correct"); // Verify similarity is bounded - assert!(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity should be bounded"); + assert!((-1.0..=1.0).contains(&similarity), "Cosine similarity should be bounded"); } #[test] @@ -904,14 +906,14 @@ mod conversion_tests { let distance = 100.0_f64; let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); let ratio: f64 = (1000.0 - 100.0) / 1000.0; - let expected: f64 = ratio.max(0.0).min(1.0); + let expected: f64 = ratio.clamp(0.0, 1.0); assert!((similarity - expected).abs() < 1e-10, "Small dot product distance conversion should be correct"); // Test medium distance let distance = 500.0_f64; let similarity = convert_distance_to_similarity(distance, SimilarityMetric::DotProduct); let ratio: f64 = (1000.0 - 500.0) / 1000.0; - let expected: f64 = ratio.max(0.0).min(1.0); + let expected: f64 = ratio.clamp(0.0, 1.0); assert!((similarity - expected).abs() < 1e-10, "Medium dot product distance conversion should be correct"); // Test maximum distance @@ -923,7 +925,7 @@ mod conversion_tests { let distances = vec![0.0_f64, 100.0, 500.0, 1000.0, 1500.0, 2000.0]; for dist in distances { let sim = convert_distance_to_similarity(dist, SimilarityMetric::DotProduct); - assert!(sim >= 0.0 && sim <= 1.0, "DotProduct similarity should be in [0, 1]"); + assert!((0.0..=1.0).contains(&sim), "DotProduct similarity should be in [0, 1]"); } } diff --git a/src/lib.rs b/src/lib.rs index 4740c58..e9a9b3e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -134,6 +134,7 @@ pub use embeddings::{EmbeddingGenerator, EmbeddingFunction}; pub use client::{VectorLiteClient, Collection, Settings, IndexType}; pub use server::{create_app, start_server}; pub use persistence::{PersistenceError, save_collection_to_file, load_collection_from_file}; +pub use errors::{VectorLiteError, VectorLiteResult}; use serde::{Serialize, Deserialize}; @@ -228,7 +229,7 @@ pub trait VectorIndex { fn delete(&mut self, id: u64) -> Result<(), String>; /// Search for the k most similar vectors - fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> Vec; + fn search(&self, query: &[f64], k: usize, similarity_metric: SimilarityMetric) -> VectorLiteResult>; /// Get the number of vectors in the index fn len(&self) -> usize; @@ -289,7 +290,7 @@ impl VectorIndex for VectorIndexWrapper { } } - fn search(&self, query: &[f64], k: usize, s: SimilarityMetric) -> Vec { + fn search(&self, query: &[f64], k: usize, s: SimilarityMetric) -> VectorLiteResult> { match self { VectorIndexWrapper::Flat(index) => index.search(query, k, s), VectorIndexWrapper::HNSW(index) => index.search(query, k, s), @@ -685,7 +686,7 @@ mod tests { ]; let store = FlatIndex::new(3, vectors); let query = vec![1.0, 0.0, 0.0]; - let results = store.search(&query, 2, SimilarityMetric::Cosine); + let results = store.search(&query, 2, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 2); assert_eq!(results[0].id, 0); @@ -717,7 +718,7 @@ mod tests { // Test search through the wrapper let query = vec![1.1, 0.1, 0.1]; - let results = deserialized.search(&query, 1, SimilarityMetric::Cosine); + let results = deserialized.search(&query, 1, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].id, 1); } diff --git a/src/persistence.rs b/src/persistence.rs index 0cccccc..9474416 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -229,7 +229,7 @@ mod tests { assert_eq!(index.dimension(), 3); // Test search functionality - let results = index.search(&[1.1, 2.1, 3.1], 1, SimilarityMetric::Cosine); + let results = index.search(&[1.1, 2.1, 3.1], 1, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].id, 0); } @@ -261,7 +261,7 @@ mod tests { assert_eq!(index.dimension(), 3); // Test search - let results = index.search(&[1.1, 2.1, 3.1], 1, SimilarityMetric::Cosine); + let results = index.search(&[1.1, 2.1, 3.1], 1, SimilarityMetric::Cosine).unwrap(); assert_eq!(results.len(), 1); assert_eq!(results[0].id, 0); } From 4dcc549d975059b959c1ecee1f77374f3bfb345a Mon Sep 17 00:00:00 2001 From: mathieu Date: Tue, 28 Oct 2025 12:12:23 +1100 Subject: [PATCH 20/21] fix clippy --- README.md | 4 + src/client.rs | 4 +- src/errors.rs | 7 +- src/persistence.rs | 33 +++++-- src/server.rs | 241 +++++++++++++++++---------------------------- 5 files changed, 125 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index 667c721..6dceca8 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Rust](https://img.shields.io/badge/rust-1.80%2B-orange.svg)](https://www.rust-lang.org) [![Tests](https://github.com/mmailhos/vectorlite/actions/workflows/rust.yml/badge.svg?branch=main)](https://github.com/mmailhos/vectorlite/actions) +[![OpenAPI](https://img.shields.io/badge/OpenAPI-3.0.3-green.svg)](docs/openapi.yaml) **A tiny, in-process Rust vector store with built-in embeddings for sub-millisecond semantic search.** @@ -74,8 +75,11 @@ docker build \ | **Save collection** | `POST /collections/{name}/save` | `{"file_path": "./collection.vlc"}` | | **Load collection** | `POST /collections/load` | `{"file_path": "./collection.vlc", "collection_name": "restored"}` | + ## Index Types +VectorLite supports 2 indexes: **Flat** and **HNSW**. + | Index | Search Complexity | Insert | Use Case | | -------- | ----------------- | -------- | ------------------------------------- | | **Flat** | O(n) | O(1) | Small datasets (<10K) or exact search | diff --git a/src/client.rs b/src/client.rs index 6364fa1..8a98b14 100644 --- a/src/client.rs +++ b/src/client.rs @@ -830,10 +830,10 @@ mod tests { fn test_collection_load_nonexistent_file() { let temp_dir = tempfile::TempDir::new().unwrap(); let file_path = temp_dir.path().join("nonexistent.vlc"); - + let result = Collection::load_from_file(&file_path); assert!(result.is_err()); - assert!(matches!(result.unwrap_err(), PersistenceError::Io(_))); + assert!(matches!(result.unwrap_err(), PersistenceError::FileNotFound(_))); } #[test] diff --git a/src/errors.rs b/src/errors.rs index 51289f2..db51c89 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -42,7 +42,7 @@ pub enum VectorLiteError { MetricMismatch { requested: crate::SimilarityMetric, index: crate::SimilarityMetric }, /// Metric required for HNSW index but not provided - #[error("HNSW index requires an explicit similarity metric. Metric must be specified when creating HNSW collections.")] + #[error("HNSW index requires an explicit similarity metric. Add field 'metric' with one of the following: ['cosine', 'euclidean', 'manhattan', 'dotproduct'] ")] MetricRequired, /// Embedding generation error @@ -81,7 +81,10 @@ impl VectorLiteError { VectorLiteError::MetricMismatch { .. } => StatusCode::BAD_REQUEST, VectorLiteError::MetricRequired => StatusCode::BAD_REQUEST, VectorLiteError::EmbeddingError(_) => StatusCode::INTERNAL_SERVER_ERROR, - VectorLiteError::PersistenceError(_) => StatusCode::INTERNAL_SERVER_ERROR, + VectorLiteError::PersistenceError(e) => match e { + crate::persistence::PersistenceError::FileNotFound(_) => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, VectorLiteError::LockError(_) => StatusCode::INTERNAL_SERVER_ERROR, VectorLiteError::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, } diff --git a/src/persistence.rs b/src/persistence.rs index 9474416..4a52623 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -35,21 +35,30 @@ use crate::{VectorIndexWrapper, Collection, VectorIndex}; #[derive(Error, Debug)] pub enum PersistenceError { #[error("IO error: {0}")] - Io(#[from] std::io::Error), - + Io(std::io::Error), + + #[error("File not found: {0}")] + FileNotFound(String), + #[error("Serialization error: {0}")] Serialization(#[from] serde_json::Error), - + #[error("Invalid file format: {0}")] InvalidFormat(String), - + #[error("Version mismatch: expected {expected}, got {actual}")] VersionMismatch { expected: String, actual: String }, - + #[error("Collection error: {0}")] Collection(String), } +impl From for PersistenceError { + fn from(error: std::io::Error) -> Self { + PersistenceError::Io(error) + } +} + /// File header containing version and format information #[derive(Debug, Serialize, Deserialize)] pub struct FileHeader { @@ -138,9 +147,15 @@ pub fn save_collection_to_file(collection: &Collection, path: &Path) -> Result<( /// Load a collection from a file pub fn load_collection_from_file(path: &Path) -> Result { - let json_data = fs::read_to_string(path)?; + let json_data = fs::read_to_string(path).map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + PersistenceError::FileNotFound(path.display().to_string()) + } else { + PersistenceError::Io(e) + } + })?; let collection_data: CollectionData = serde_json::from_str(&json_data)?; - + // Validate version compatibility if collection_data.header.version != "1.0.0" { return Err(PersistenceError::VersionMismatch { @@ -148,7 +163,7 @@ pub fn load_collection_from_file(path: &Path) -> Result Result VectorLiteResult { } } +// Implement IntoResponse for VectorLiteError to enable automatic error responses +impl IntoResponse for VectorLiteError { + fn into_response(self) -> Response { + let status = self.status_code(); + let error_message = self.to_string(); + + let body = Json(ErrorResponse { + message: error_message, + }); + + (status, body).into_response() + } +} + // Handlers async fn health_check() -> Json { Json(serde_json::json!({ @@ -175,8 +188,8 @@ async fn health_check() -> Json { async fn list_collections( State(state): State, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for list_collections".to_string()))?; let collections = client.list_collections(); Ok(Json(ListCollectionsResponse { collections, @@ -186,87 +199,58 @@ async fn list_collections( async fn create_collection( State(state): State, Json(payload): Json, -) -> Result, StatusCode> { - let index_type = match parse_index_type(&payload.index_type) { - Ok(t) => t, - Err(e) => { - return Err(e.status_code()); - } - }; +) -> Result, VectorLiteError> { + let index_type = parse_index_type(&payload.index_type)?; // Parse metric - optional for Flat index, required for HNSW let metric = if payload.metric.is_empty() { None // No metric specified } else { - match parse_similarity_metric(&payload.metric) { - Ok(m) => Some(m), - Err(e) => { - return Err(e.status_code()); - } - } + Some(parse_similarity_metric(&payload.metric)?) }; - let mut client = state.write().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.create_collection(&payload.name, index_type, metric) { - Ok(_) => { - info!("Created collection: {}", payload.name); - Ok(Json(CreateCollectionResponse { - name: payload.name, - })) - } - Err(e) => { - error!("Failed to create collection '{}': {}", payload.name, e); - Err(e.status_code()) - } - } + let mut client = state.write().map_err(|_| VectorLiteError::LockError("Failed to acquire write lock for create_collection".to_string()))?; + client.create_collection(&payload.name, index_type, metric)?; + info!("Created collection: {}", payload.name); + Ok(Json(CreateCollectionResponse { + name: payload.name, + })) } async fn get_collection_info( State(state): State, Path(collection_name): Path, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.get_collection_info(&collection_name) { - Ok(info) => Ok(Json(CollectionInfoResponse { - info: Some(info), - })), - Err(e) => Err(e.status_code()), - } +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for get_collection_info".to_string()))?; + let info = client.get_collection_info(&collection_name)?; + Ok(Json(CollectionInfoResponse { + info: Some(info), + })) } async fn delete_collection( State(state): State, Path(collection_name): Path, -) -> Result, StatusCode> { - let mut client = state.write().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.delete_collection(&collection_name) { - Ok(_) => { - info!("Deleted collection: {}", collection_name); - Ok(Json(CreateCollectionResponse { - name: collection_name, - })) - } - Err(e) => Err(e.status_code()), - } +) -> Result, VectorLiteError> { + let mut client = state.write().map_err(|_| VectorLiteError::LockError("Failed to acquire write lock for delete_collection".to_string()))?; + client.delete_collection(&collection_name)?; + info!("Deleted collection: {}", collection_name); + Ok(Json(CreateCollectionResponse { + name: collection_name, + })) } async fn add_text( State(state): State, Path(collection_name): Path, Json(payload): Json, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.add_text_to_collection(&collection_name, &payload.text, payload.metadata) { - Ok(id) => { - info!("Added text to collection '{}' with ID: {}", collection_name, id); - Ok(Json(AddTextResponse { - id: Some(id), - })) - } - Err(e) => { - Err(e.status_code()) - } - } +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for add_text".to_string()))?; + let id = client.add_text_to_collection(&collection_name, &payload.text, payload.metadata)?; + info!("Added text to collection '{}' with ID: {}", collection_name, id); + Ok(Json(AddTextResponse { + id: Some(id), + })) } @@ -275,144 +259,99 @@ async fn search_text( State(state): State, Path(collection_name): Path, Json(payload): Json, -) -> Result, StatusCode> { +) -> Result, VectorLiteError> { let k = payload.k.unwrap_or(10); let similarity_metric = match payload.similarity_metric { - Some(metric) => match parse_similarity_metric(&metric) { - Ok(m) => Some(m), - Err(e) => { - return Err(e.status_code()); - } - }, + Some(metric) => Some(parse_similarity_metric(&metric)?), None => None, // No metric specified - will auto-detect }; - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.search_text_in_collection(&collection_name, &payload.query, k, similarity_metric) { - Ok(results) => { - info!("Search completed for collection '{}' with {} results", collection_name, results.len()); - Ok(Json(SearchResponse { - results: Some(results), - })) - } - Err(e) => Err(e.status_code()), - } + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for search_text".to_string()))?; + let results = client.search_text_in_collection(&collection_name, &payload.query, k, similarity_metric)?; + info!("Search completed for collection '{}' with {} results", collection_name, results.len()); + Ok(Json(SearchResponse { + results: Some(results), + })) } async fn get_vector( State(state): State, Path((collection_name, vector_id)): Path<(String, u64)>, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.get_vector_from_collection(&collection_name, vector_id) { - Ok(Some(vector)) => { - Ok(Json(serde_json::json!({ - "vector": vector - }))) - } - Ok(None) => { - Err(StatusCode::NOT_FOUND) - } - Err(e) => { - Err(e.status_code()) - } - } +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for get_vector".to_string()))?; + let vector = client.get_vector_from_collection(&collection_name, vector_id)? + .ok_or(VectorLiteError::VectorNotFound { id: vector_id })?; + Ok(Json(serde_json::json!({ + "vector": vector + }))) } async fn delete_vector( State(state): State, Path((collection_name, vector_id)): Path<(String, u64)>, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - match client.delete_from_collection(&collection_name, vector_id) { - Ok(_) => { - info!("Deleted vector {} from collection '{}'", vector_id, collection_name); - Ok(Json(serde_json::json!({}))) - } - Err(e) => { - Err(e.status_code()) - } - } +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for delete_vector".to_string()))?; + client.delete_from_collection(&collection_name, vector_id)?; + info!("Deleted vector {} from collection '{}'", vector_id, collection_name); + Ok(Json(serde_json::json!({}))) } async fn save_collection( State(state): State, Path(collection_name): Path, Json(payload): Json, -) -> Result, StatusCode> { - let client = state.read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - +) -> Result, VectorLiteError> { + let client = state.read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for save_collection".to_string()))?; + // Get the collection - let collection = match client.get_collection(&collection_name) { - Some(collection) => collection, - None => { - return Err(StatusCode::NOT_FOUND); - } - }; + let collection = client.get_collection(&collection_name) + .ok_or_else(|| VectorLiteError::CollectionNotFound { name: collection_name.clone() })?; // Convert file path to PathBuf let file_path = PathBuf::from(&payload.file_path); - + // Save the collection - match collection.save_to_file(&file_path) { - Ok(_) => { - info!("Saved collection '{}' to file: {}", collection_name, payload.file_path); - Ok(Json(SaveCollectionResponse { - file_path: Some(payload.file_path), - })) - } - Err(_) => { - Err(StatusCode::INTERNAL_SERVER_ERROR) - } - } + collection.save_to_file(&file_path)?; + info!("Saved collection '{}' to file: {}", collection_name, payload.file_path); + Ok(Json(SaveCollectionResponse { + file_path: Some(payload.file_path), + })) } async fn load_collection( State(state): State, Json(payload): Json, -) -> Result, StatusCode> { +) -> Result, VectorLiteError> { // Convert file path to PathBuf let file_path = PathBuf::from(&payload.file_path); - + // Load the collection from file - let collection = match crate::Collection::load_from_file(&file_path) { - Ok(collection) => collection, - Err(e) => { - // Check if it's a file not found error - if let crate::persistence::PersistenceError::Io(io_err) = &e - && io_err.kind() == std::io::ErrorKind::NotFound { - return Err(VectorLiteError::FileNotFound(format!("File not found: {}", payload.file_path)).status_code()); - } - return Err(VectorLiteError::from(e).status_code()); - } - }; + let collection = crate::Collection::load_from_file(&file_path)?; // Determine the collection name to use let collection_name = payload.collection_name.unwrap_or_else(|| collection.name().to_string()); - + // Add the collection to the client - let mut client = state.write().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - + let mut client = state.write().map_err(|_| VectorLiteError::LockError("Failed to acquire write lock for load_collection".to_string()))?; + // Check if collection already exists if client.has_collection(&collection_name) { - return Err(StatusCode::CONFLICT); + return Err(VectorLiteError::CollectionAlreadyExists { name: collection_name }); } // Extract the index from the loaded collection let index = { - let index_guard = collection.index_read().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let index_guard = collection.index_read().map_err(|_| VectorLiteError::LockError("Failed to acquire read lock for collection index".to_string()))?; (*index_guard).clone() }; - + // Create a new collection with the loaded data let new_collection = crate::Collection::new(collection_name.clone(), index); - + // Add the collection to the client - if client.add_collection(new_collection).is_err() { - return Err(StatusCode::INTERNAL_SERVER_ERROR); - } - + client.add_collection(new_collection)?; + info!("Loaded collection '{}' from file: {}", collection_name, payload.file_path); Ok(Json(LoadCollectionResponse { collection_name: Some(collection_name), From b7f1989562c0947d4ce7045ce5c65f3a72644e13 Mon Sep 17 00:00:00 2001 From: mathieu Date: Tue, 28 Oct 2025 12:32:59 +1100 Subject: [PATCH 21/21] add openapi specs --- README.md | 6 +- docs/openapi.yaml | 839 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 843 insertions(+), 2 deletions(-) create mode 100644 docs/openapi.yaml diff --git a/README.md b/README.md index 6dceca8..1c6674f 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ docker build \ ``` ## HTTP API Overview + | Operation | Method & Endpoint | Body | | --------------------- | ----------------------------------------- | ------------------------------------------------------------------ | | **Health** | `GET /health` | – | @@ -87,8 +88,6 @@ VectorLite supports 2 indexes: **Flat** and **HNSW**. See [Hierarchical Navigable Small World](https://arxiv.org/abs/1603.09320). -Note: Flat indices support all metrics dynamically. HNSW index must be created with a default distance metric (`cosine`, `euclidean`, `manhattan` or `dotproduct`). - ### Configuration profiles for HNSW | Profile | Features | Use Case | @@ -103,6 +102,9 @@ cargo build --features memory-optimized ### Similarity Metrics + +A flat index is the most flexible as it allows for all search metric operations. On the other hand, the HNSW index is specifically optimised for a specific distance metric, which will be used for all search operations. When creating a HNSW index, provide a `metric` value with one of: `cosine`, `euclidean`, `manhattan` or `dotproduct`. + - **Cosine**: Default for normalized embeddings, scale-invariant - **Euclidean**: Geometric distance, sensitive to vector magnitude - **Manhattan**: L1 norm, robust to outliers diff --git a/docs/openapi.yaml b/docs/openapi.yaml new file mode 100644 index 0000000..3736909 --- /dev/null +++ b/docs/openapi.yaml @@ -0,0 +1,839 @@ +openapi: 3.0.3 +info: + title: VectorLite API + description: | + A high-performance, in-memory vector database optimized for AI agent and edge workloads. + VectorLite provides sub-millisecond semantic search with built-in embedding generation. + + ## Features + + - **Sub-millisecond search**: In-memory HNSW or flat search + - **Built-in embeddings**: Runs all-MiniLM-L6-v2 locally using Candle + - **Thread-safe**: Concurrent read access with atomic ID generation + - **Persistence**: Save and restore collections to/from disk + - **Flexible metrics**: Cosine, Euclidean, Manhattan, and Dot Product similarity + + ## Index Types + + | Index | Search Complexity | Use Case | + |-------|------------------|----------| + | **Flat** | O(n) | Small datasets (<10K) or exact search | + | **HNSW** | O(log n) | Larger datasets or approximate search | + + ## Similarity Metrics + + | Metric | Description | Range | + |--------|-------------|-------| + | **Cosine** | Scale-invariant, good for normalized embeddings | [-1, 1] | + | **Euclidean** | Geometric distance, sensitive to magnitude | [0, 1] | + | **Manhattan** | L1 norm, robust to outliers | [0, 1] | + | **Dot Product** | Raw similarity, requires consistent scaling | unbounded | + version: 0.1.5 + contact: + name: VectorLite Support + url: https://github.com/mmailhos/vectorlite + +servers: + - url: http://localhost:3001 + description: Local development server + - url: http://localhost:3002 + description: Alternative port + +tags: + - name: Health + description: Health check endpoints + - name: Collections + description: Collection management operations + - name: Vectors + description: Vector operations (add, search, get, delete) + - name: Persistence + description: Save and load collection operations + +paths: + /health: + get: + tags: + - Health + summary: Health check + description: Returns the health status of the server + operationId: healthCheck + responses: + '200': + description: Server is healthy + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: healthy + service: + type: string + example: vectorlite + + /collections: + get: + tags: + - Collections + summary: List all collections + description: Returns a list of all collection names + operationId: listCollections + responses: + '200': + description: List of collections + content: + application/json: + schema: + $ref: '#/components/schemas/ListCollectionsResponse' + + post: + tags: + - Collections + summary: Create a new collection + description: Creates a new collection with the specified index type and similarity metric + operationId: createCollection + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCollectionRequest' + examples: + flat_index: + summary: Create a flat index + value: + name: small_docs + index_type: flat + metric: "" + hnsw_cosine: + summary: Create HNSW index with cosine metric + value: + name: large_docs + index_type: hnsw + metric: cosine + hnsw_euclidean: + summary: Create HNSW index with euclidean metric + value: + name: geo_docs + index_type: hnsw + metric: euclidean + responses: + '200': + description: Collection created successfully + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCollectionResponse' + '400': + description: Bad request (invalid index type, metric, or missing required metric) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + invalid_index: + value: + message: "Invalid index type: invalid. Must be 'flat' or 'hnsw'" + invalid_metric: + value: + message: "Invalid similarity metric: invalid. Must be 'cosine', 'euclidean', 'manhattan', or 'dotproduct'" + metric_required: + value: + message: "HNSW index requires an explicit similarity metric. Add field 'metric' with one of the following: ['cosine', 'euclidean', 'manhattan', 'dotproduct']" + '409': + description: Collection already exists + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + example: + message: "Collection 'docs' already exists" + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/{name}: + get: + tags: + - Collections + summary: Get collection information + description: Returns detailed information about a specific collection + operationId: getCollectionInfo + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + responses: + '200': + description: Collection information + content: + application/json: + schema: + $ref: '#/components/schemas/CollectionInfoResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + example: + message: "Collection 'my_docs' not found" + + delete: + tags: + - Collections + summary: Delete a collection + description: Deletes a collection and all its vectors + operationId: deleteCollection + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + responses: + '200': + description: Collection deleted successfully + content: + application/json: + schema: + $ref: '#/components/schemas/CreateCollectionResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/{name}/text: + post: + tags: + - Vectors + summary: Add text to collection + description: | + Adds text to a collection. The text is automatically converted to an embedding, + and the vector is added to the collection. Returns the ID of the newly created vector. + + **Note**: The embedding is generated using the all-MiniLM-L6-v2 model (384 dimensions by default). + operationId: addText + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AddTextRequest' + examples: + simple: + summary: Add text without metadata + value: + text: "Hello, world! This is a sample document." + with_metadata: + summary: Add text with metadata + value: + text: "AI agents are revolutionizing software development" + metadata: + author: "John Doe" + tags: + - ai + - agents + - ml + published: "2024-01-15" + views: 1234 + responses: + '200': + description: Text added successfully + content: + application/json: + schema: + $ref: '#/components/schemas/AddTextResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '400': + description: Bad request (dimension mismatch or embedding generation error) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + dimension_mismatch: + value: + message: "Vector dimension mismatch: expected 384, got 256" + '409': + description: Duplicate vector ID + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error (e.g., embedding generation failed) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/{name}/search/text: + post: + tags: + - Vectors + summary: Search by text query + description: | + Searches the collection for vectors similar to the query text. + The query is automatically converted to an embedding, then searched using + the collection's index and similarity metric. + operationId: searchText + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SearchTextRequest' + examples: + default: + summary: Search with default settings + value: + query: "machine learning" + k: 5 + custom_metric: + summary: Search with custom similarity metric + value: + query: "artificial intelligence" + k: 10 + similarity_metric: euclidean + max_results: + summary: Get top 20 results + value: + query: "neural networks" + k: 20 + responses: + '200': + description: Search completed successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SearchResponse' + '400': + description: Bad request (invalid similarity metric or metric mismatch) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + invalid_metric: + value: + message: "Invalid similarity metric: invalid. Must be 'cosine', 'euclidean', 'manhattan', or 'dotproduct'" + metric_mismatch: + value: + message: "Metric mismatch: search requested Euclidean but index was built for Cosine" + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/{name}/vectors/{id}: + get: + tags: + - Vectors + summary: Get vector by ID + description: Retrieves a specific vector from a collection by its ID + operationId: getVector + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + - name: id + in: path + required: true + description: The ID of the vector + schema: + type: integer + format: int64 + example: 123 + responses: + '200': + description: Vector retrieved successfully + content: + application/json: + schema: + type: object + properties: + vector: + $ref: '#/components/schemas/Vector' + '404': + description: Vector or collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + delete: + tags: + - Vectors + summary: Delete a vector + description: Deletes a specific vector from a collection by its ID + operationId: deleteVector + parameters: + - name: name + in: path + required: true + description: The name of the collection + schema: + type: string + example: my_docs + - name: id + in: path + required: true + description: The ID of the vector + schema: + type: integer + format: int64 + example: 123 + responses: + '200': + description: Vector deleted successfully + content: + application/json: + schema: + type: object + '404': + description: Vector or collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/{name}/save: + post: + tags: + - Persistence + summary: Save collection to file + description: | + Saves the entire collection to disk in a binary format (.vlc file). + This includes all vectors, the index structure, and metadata. + + **Note**: The file will be created if it doesn't exist, and overwritten if it does. + operationId: saveCollection + parameters: + - name: name + in: path + required: true + description: The name of the collection to save + schema: + type: string + example: my_docs + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SaveCollectionRequest' + example: + file_path: "./backups/my_docs.vlc" + responses: + '200': + description: Collection saved successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SaveCollectionResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error (e.g., disk I/O failure) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + /collections/load: + post: + tags: + - Persistence + summary: Load collection from file + description: | + Loads a collection from a previously saved file (.vlc format). + The collection name can be specified, or it will use the name from the saved file. + + **Note**: This operation will fail if a collection with the same name already exists. + You must delete the existing collection first if you want to replace it. + operationId: loadCollection + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LoadCollectionRequest' + examples: + load_with_name: + summary: Load with custom collection name + value: + file_path: "./backups/my_docs.vlc" + collection_name: restored_docs + load_without_name: + summary: Load with original collection name + value: + file_path: "./backups/my_docs.vlc" + responses: + '200': + description: Collection loaded successfully + content: + application/json: + schema: + $ref: '#/components/schemas/LoadCollectionResponse' + '404': + description: File not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + example: + message: "File not found: ./backups/my_docs.vlc" + '409': + description: Collection already exists + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + example: + message: "Collection 'restored_docs' already exists" + '500': + description: Internal server error (e.g., invalid file format) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + +components: + schemas: + # Request schemas + CreateCollectionRequest: + type: object + required: + - name + - index_type + properties: + name: + type: string + description: Unique name for the collection + example: my_docs + index_type: + type: string + enum: [flat, hnsw] + description: Type of index to use (flat for exact search, hnsw for approximate) + example: hnsw + metric: + type: string + default: "" + description: Similarity metric (cosine, euclidean, manhattan, dotproduct). Empty string for default. + example: cosine + + AddTextRequest: + type: object + required: + - text + properties: + text: + type: string + description: The text to add to the collection + example: "This is a sample document about machine learning." + metadata: + type: object + description: Optional metadata associated with this text (any JSON object) + additionalProperties: true + example: + author: "John Doe" + category: "machine-learning" + tags: + - ai + - ml + - research + + SearchTextRequest: + type: object + required: + - query + properties: + query: + type: string + description: The search query text + example: "neural networks" + k: + type: integer + format: int32 + default: 10 + description: Number of results to return + minimum: 1 + maximum: 1000 + example: 5 + similarity_metric: + type: string + enum: [cosine, euclidean, manhattan, dotproduct] + description: Override the collection's default similarity metric for this search + example: cosine + + SaveCollectionRequest: + type: object + required: + - file_path + properties: + file_path: + type: string + description: Path where the collection should be saved (including .vlc extension) + example: "./backups/my_docs.vlc" + + LoadCollectionRequest: + type: object + required: + - file_path + properties: + file_path: + type: string + description: Path to the collection file to load + example: "./backups/my_docs.vlc" + collection_name: + type: string + description: Optional name for the loaded collection. If not provided, uses the name from the file. + example: restored_docs + + # Response schemas + CreateCollectionResponse: + type: object + properties: + name: + type: string + description: Name of the created collection + example: my_docs + + AddTextResponse: + type: object + properties: + id: + type: integer + format: int64 + nullable: true + description: ID of the newly added vector + example: 42 + + SearchResponse: + type: object + properties: + results: + type: array + nullable: true + description: Array of search results sorted by similarity (highest first) + items: + $ref: '#/components/schemas/SearchResult' + + SearchResult: + type: object + properties: + id: + type: integer + format: int64 + description: ID of the matching vector + example: 42 + score: + type: number + format: float + description: Similarity score (higher is more similar) + example: 0.9234 + text: + type: string + description: Original text that was embedded + example: "This document is about machine learning algorithms" + metadata: + type: object + nullable: true + description: Metadata associated with this vector (if any) + additionalProperties: true + example: + author: "John Doe" + category: "machine-learning" + + Vector: + type: object + properties: + id: + type: integer + format: int64 + description: Unique identifier for the vector + example: 123 + values: + type: array + items: + type: number + format: float + description: The embedding vector values (typically 384 dimensions for all-MiniLM-L6-v2) + example: [0.1, 0.2, -0.3, 0.4, 0.5, -0.6] + text: + type: string + description: The original text that was embedded to create this vector + example: "Sample document text" + metadata: + type: object + nullable: true + description: Optional metadata associated with this vector (any JSON object) + additionalProperties: true + example: + author: "John Doe" + tags: + - tutorial + - example + + ListCollectionsResponse: + type: object + properties: + collections: + type: array + items: + type: string + description: List of collection names + example: [my_docs, other_collection] + + CollectionInfoResponse: + type: object + properties: + info: + nullable: true + $ref: '#/components/schemas/CollectionInfo' + + CollectionInfo: + type: object + properties: + name: + type: string + description: Name of the collection + example: my_docs + count: + type: integer + format: int32 + description: Number of vectors in the collection + example: 1234 + is_empty: + type: boolean + description: Whether the collection is empty + example: false + dimension: + type: integer + format: int32 + description: Dimension of vectors in this collection (typically 384 for all-MiniLM-L6-v2) + example: 384 + + SaveCollectionResponse: + type: object + properties: + file_path: + type: string + nullable: true + description: Path where the collection was saved + example: "./backups/my_docs.vlc" + + LoadCollectionResponse: + type: object + properties: + collection_name: + type: string + nullable: true + description: Name of the loaded collection + example: restored_docs + + ErrorResponse: + type: object + properties: + message: + type: string + description: Error message + example: "Collection 'my_docs' not found" + + responses: + NotFound: + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + BadRequest: + description: Bad request (validation error) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + Conflict: + description: Resource conflict (e.g., already exists) + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + InternalServerError: + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' +