From affcaef55638ba6bffe9079ab1ff70b75ca508a0 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 21 Jul 2025 11:42:46 +0200 Subject: [PATCH 01/62] Use Hannoy instead of arroy --- Cargo.lock | 84 ++++--- .../index-scheduler/src/index_mapper/mod.rs | 6 +- .../tests/vector/binary_quantized.rs | 2 +- crates/meilisearch/tests/vector/mod.rs | 2 +- crates/meilisearch/tests/vector/settings.rs | 2 +- crates/meilitool/src/main.rs | 12 +- crates/meilitool/src/upgrade/v1_11.rs | 17 +- crates/milli/Cargo.toml | 2 +- crates/milli/src/error.rs | 31 +-- crates/milli/src/index.rs | 28 +-- crates/milli/src/lib.rs | 2 +- crates/milli/src/progress.rs | 82 +++--- crates/milli/src/search/new/vector_sort.rs | 5 +- crates/milli/src/search/similar.rs | 4 +- crates/milli/src/update/clear_documents.rs | 4 +- .../milli/src/update/index_documents/mod.rs | 8 +- .../src/update/index_documents/transform.rs | 22 +- .../src/update/index_documents/typed_chunk.rs | 5 +- crates/milli/src/update/new/channel.rs | 64 ++--- crates/milli/src/update/new/indexer/mod.rs | 40 +-- crates/milli/src/update/new/indexer/write.rs | 40 +-- .../milli/src/update/new/vector_document.rs | 8 +- crates/milli/src/update/upgrade/v1_14.rs | 15 +- crates/milli/src/vector/composite.rs | 17 +- crates/milli/src/vector/mod.rs | 234 +++++++++--------- 25 files changed, 380 insertions(+), 356 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc96152a6..378d08124 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -442,28 +442,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arroy" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08e6111f351d004bd13e95ab540721272136fd3218b39d3ec95a2ea1c4e6a0a6" -dependencies = [ - "bytemuck", - "byteorder", - "enum-iterator", - "heed", - "memmap2", - "nohash", - "ordered-float 4.6.0", - "page_size", - "rand 0.8.5", - "rayon", - "roaring", - "tempfile", - "thiserror 2.0.12", - "tracing", -] - [[package]] name = "assert-json-diff" version = "2.0.2" @@ -2601,6 +2579,31 @@ dependencies = [ "rand_distr", ] +[[package]] +name = "hannoy" +version = "0.0.1" +source = "git+https://github.com/nnethercott/hannoy?tag=v0.0.1#d51750cd5612b6875375f5f4ad3928c87d55ee38" +dependencies = [ + "bytemuck", + "byteorder", + "enum-iterator", + "hashbrown 0.15.4", + "heed", + "memmap2", + "min-max-heap", + "nohash", + "ordered-float 5.0.0", + "page_size", + "papaya", + "rand 0.8.5", + "rayon", + "roaring", + "tempfile", + "thiserror 2.0.12", + "tinyvec", + "tracing", +] + [[package]] name = "hash32" version = "0.3.1" @@ -3924,7 +3927,6 @@ name = "milli" version = "1.19.0" dependencies = [ "allocator-api2 0.3.0", - "arroy", "bbqueue", "big_s", "bimap", @@ -3952,6 +3954,7 @@ dependencies = [ "fxhash", "geoutils", "grenad", + "hannoy", "hashbrown 0.15.4", "heed", "hf-hub", @@ -4022,6 +4025,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "min-max-heap" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4373,15 +4382,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ordered-float" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "5.0.0" @@ -4413,6 +4413,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "papaya" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" +dependencies = [ + "equivalent", + "seize", +] + [[package]] name = "parking_lot" version = "0.12.4" @@ -5464,6 +5474,16 @@ dependencies = [ "time", ] +[[package]] +name = "seize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "semver" version = "1.0.26" diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index d578b03dd..7014f87cb 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -143,10 +143,10 @@ impl IndexStats { /// /// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`. pub fn new(index: &Index, rtxn: &RoTxn) -> milli::Result { - let arroy_stats = index.arroy_stats(rtxn)?; + let hannoy_stats = index.hannoy_stats(rtxn)?; Ok(IndexStats { - number_of_embeddings: Some(arroy_stats.number_of_embeddings), - number_of_embedded_documents: Some(arroy_stats.documents.len()), + number_of_embeddings: Some(hannoy_stats.number_of_embeddings), + number_of_embedded_documents: Some(hannoy_stats.documents.len()), documents_database_stats: index.documents_stats(rtxn)?.unwrap_or_default(), number_of_documents: None, database_size: index.on_disk_size()?, diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index adb0da441..c5a7c9244 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -320,7 +320,7 @@ async fn binary_quantize_clear_documents() { } "###); - // Make sure the arroy DB has been cleared + // Make sure the hannoy DB has been cleared let (documents, _code) = index.search_post(json!({ "hybrid": { "embedder": "manual" }, "vector": [1, 1, 1] })).await; snapshot!(documents, @r#" diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 3c08b9e03..9eb7deff6 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -684,7 +684,7 @@ async fn clear_documents() { } "###); - // Make sure the arroy DB has been cleared + // Make sure the hannoy DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "manual"} })).await; snapshot!(documents, @r#" diff --git a/crates/meilisearch/tests/vector/settings.rs b/crates/meilisearch/tests/vector/settings.rs index d26174faf..8ace8f092 100644 --- a/crates/meilisearch/tests/vector/settings.rs +++ b/crates/meilisearch/tests/vector/settings.rs @@ -236,7 +236,7 @@ async fn reset_embedder_documents() { } "###); - // Make sure the arroy DB has been cleared + // Make sure the hannoy DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "default"} })).await; snapshot!(json_string!(documents), @r###" diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 170bbdcc8..439bba015 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -142,8 +142,8 @@ enum Command { #[derive(Clone, ValueEnum)] enum IndexPart { - /// Will make the arroy index hot. - Arroy, + /// Will make the hannoy index hot. + Hannoy, } fn main() -> anyhow::Result<()> { @@ -658,12 +658,12 @@ fn hair_dryer( let rtxn = index.read_txn()?; for part in index_parts { match part { - IndexPart::Arroy => { + IndexPart::Hannoy => { let mut count = 0; - let total = index.vector_arroy.len(&rtxn)?; - eprintln!("Hair drying arroy for {uid}..."); + let total = index.vector_hannoy.len(&rtxn)?; + eprintln!("Hair drying hannoy for {uid}..."); for (i, result) in index - .vector_arroy + .vector_hannoy .remap_types::() .iter(&rtxn)? .enumerate() diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index 76d2fc24f..385487b89 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -68,7 +68,7 @@ pub fn v1_10_to_v1_11( ) })?; let index_read_database = - try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_ARROY) + try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_HANNOY) .with_context(|| format!("while updating date format for index `{uid}`"))?; let mut index_wtxn = index_env.write_txn().with_context(|| { @@ -79,15 +79,16 @@ pub fn v1_10_to_v1_11( })?; let index_write_database = - try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY) + try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_HANNOY) .with_context(|| format!("while updating date format for index `{uid}`"))?; - meilisearch_types::milli::arroy::upgrade::cosine_from_0_4_to_0_5( - &index_rtxn, - index_read_database.remap_types(), - &mut index_wtxn, - index_write_database.remap_types(), - )?; + // meilisearch_types::milli::hannoy::upgrade::cosine_from_0_4_to_0_5( + // &index_rtxn, + // index_read_database.remap_types(), + // &mut index_wtxn, + // index_write_database.remap_types(), + // )?; + unimplemented!("Hannoy doesn't support upgrading"); index_wtxn.commit()?; } diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 28da78835..812ac4376 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -87,7 +87,7 @@ rhai = { version = "1.22.2", features = [ "no_time", "sync", ] } -arroy = "0.6.1" +hannoy = { git = "https://github.com/nnethercott/hannoy", tag = "v0.0.1" } rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 76ad3fda0..2769284aa 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -76,7 +76,7 @@ pub enum InternalError { #[error("Cannot upgrade to the following version: v{0}.{1}.{2}.")] CannotUpgradeToVersion(u32, u32, u32), #[error(transparent)] - ArroyError(#[from] arroy::Error), + HannoyError(#[from] hannoy::Error), #[error(transparent)] VectorEmbeddingError(#[from] crate::vector::Error), } @@ -419,23 +419,24 @@ impl From for Error { } } -impl From for Error { - fn from(value: arroy::Error) -> Self { +impl From for Error { + fn from(value: hannoy::Error) -> Self { match value { - arroy::Error::Heed(heed) => heed.into(), - arroy::Error::Io(io) => io.into(), - arroy::Error::InvalidVecDimension { expected, received } => { + hannoy::Error::Heed(heed) => heed.into(), + hannoy::Error::Io(io) => io.into(), + hannoy::Error::InvalidVecDimension { expected, received } => { Error::UserError(UserError::InvalidVectorDimensions { expected, found: received }) } - arroy::Error::BuildCancelled => Error::InternalError(InternalError::AbortedIndexation), - arroy::Error::DatabaseFull - | arroy::Error::InvalidItemAppend - | arroy::Error::UnmatchingDistance { .. } - | arroy::Error::NeedBuild(_) - | arroy::Error::MissingKey { .. } - | arroy::Error::MissingMetadata(_) - | arroy::Error::CannotDecodeKeyMode { .. } => { - Error::InternalError(InternalError::ArroyError(value)) + hannoy::Error::BuildCancelled => Error::InternalError(InternalError::AbortedIndexation), + hannoy::Error::DatabaseFull + | hannoy::Error::InvalidItemAppend + | hannoy::Error::UnmatchingDistance { .. } + | hannoy::Error::NeedBuild(_) + | hannoy::Error::MissingKey { .. } + | hannoy::Error::MissingMetadata(_) + | hannoy::Error::UnknownVersion { .. } + | hannoy::Error::CannotDecodeKeyMode { .. } => { + Error::InternalError(InternalError::HannoyError(value)) } } } diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 6429dabbc..4c4712c63 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -31,7 +31,7 @@ use crate::prompt::PromptData; use crate::proximity::ProximityPrecision; use crate::update::new::StdResult; use crate::vector::db::IndexEmbeddingConfigs; -use crate::vector::{ArroyStats, ArroyWrapper, Embedding}; +use crate::vector::{Embedding, HannoyStats, HannoyWrapper}; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldIdWordCountCodec, @@ -113,7 +113,7 @@ pub mod db_name { pub const FIELD_ID_DOCID_FACET_F64S: &str = "field-id-docid-facet-f64s"; pub const FIELD_ID_DOCID_FACET_STRINGS: &str = "field-id-docid-facet-strings"; pub const VECTOR_EMBEDDER_CATEGORY_ID: &str = "vector-embedder-category-id"; - pub const VECTOR_ARROY: &str = "vector-arroy"; + pub const VECTOR_HANNOY: &str = "vector-hannoy"; pub const DOCUMENTS: &str = "documents"; } const NUMBER_OF_DBS: u32 = 25; @@ -177,10 +177,10 @@ pub struct Index { /// Maps the document id, the facet field id and the strings. pub field_id_docid_facet_strings: Database, - /// Maps an embedder name to its id in the arroy store. + /// Maps an embedder name to its id in the hannoy store. pub(crate) embedder_category_id: Database, - /// Vector store based on arroy™. - pub vector_arroy: arroy::Database, + /// Vector store based on hannoy™. + pub vector_hannoy: hannoy::Database, /// Maps the document id to the document as an obkv store. pub(crate) documents: Database, @@ -237,7 +237,7 @@ impl Index { // vector stuff let embedder_category_id = env.create_database(&mut wtxn, Some(VECTOR_EMBEDDER_CATEGORY_ID))?; - let vector_arroy = env.create_database(&mut wtxn, Some(VECTOR_ARROY))?; + let vector_hannoy = env.create_database(&mut wtxn, Some(VECTOR_HANNOY))?; let documents = env.create_database(&mut wtxn, Some(DOCUMENTS))?; @@ -264,7 +264,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_arroy, + vector_hannoy, embedder_category_id, documents, }; @@ -1772,8 +1772,8 @@ impl Index { for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); let has_fragments = config.config.embedder_options.has_fragments(); - let reader = ArroyWrapper::new( - self.vector_arroy, + let reader = HannoyWrapper::new( + self.vector_hannoy, embedder_info.embedder_id, config.config.quantized(), ); @@ -1792,13 +1792,13 @@ impl Index { Ok(PrefixSettings { compute_prefixes, max_prefix_length: 4, prefix_count_threshold: 100 }) } - pub fn arroy_stats(&self, rtxn: &RoTxn<'_>) -> Result { - let mut stats = ArroyStats::default(); + pub fn hannoy_stats(&self, rtxn: &RoTxn<'_>) -> Result { + let mut stats = HannoyStats::default(); let embedding_configs = self.embedding_configs(); for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); let reader = - ArroyWrapper::new(self.vector_arroy, embedder_id, config.config.quantized()); + HannoyWrapper::new(self.vector_hannoy, embedder_id, config.config.quantized()); reader.aggregate_stats(rtxn, &mut stats)?; } Ok(stats) @@ -1842,7 +1842,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_arroy, + vector_hannoy, embedder_category_id, documents, } = self; @@ -1913,7 +1913,7 @@ impl Index { "field_id_docid_facet_strings", field_id_docid_facet_strings.stat(rtxn).map(compute_size)?, ); - sizes.insert("vector_arroy", vector_arroy.stat(rtxn).map(compute_size)?); + sizes.insert("vector_hannoy", vector_hannoy.stat(rtxn).map(compute_size)?); sizes.insert("embedder_category_id", embedder_category_id.stat(rtxn).map(compute_size)?); sizes.insert("documents", documents.stat(rtxn).map(compute_size)?); diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index 6fdae86b3..91ea87b68 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -53,7 +53,7 @@ pub use search::new::{ }; use serde_json::Value; pub use thread_pool_no_abort::{PanicCatched, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder}; -pub use {arroy, charabia as tokenizer, heed, rhai}; +pub use {charabia as tokenizer, hannoy, heed, rhai}; pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError}; pub use self::attribute_patterns::{AttributePatterns, PatternMatch}; diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index 61c61cd49..2ec4fd6de 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -98,12 +98,12 @@ impl Progress { } // TODO: ideally we should expose the progress in a way that let arroy use it directly - pub(crate) fn update_progress_from_arroy(&self, progress: arroy::WriterProgress) { - self.update_progress(progress.main); - if let Some(sub) = progress.sub { - self.update_progress(sub); - } - } + // pub(crate) fn update_progress_from_hannoy(&self, progress: hannoy::WriterProgress) { + // self.update_progress(progress.main); + // if let Some(sub) = progress.sub { + // self.update_progress(sub); + // } + // } } /// Generate the names associated with the durations and push them. @@ -277,43 +277,43 @@ impl Step for VariableNameStep { } } -impl Step for arroy::MainStep { - fn name(&self) -> Cow<'static, str> { - match self { - arroy::MainStep::PreProcessingTheItems => "pre processing the items", - arroy::MainStep::WritingTheDescendantsAndMetadata => { - "writing the descendants and metadata" - } - arroy::MainStep::RetrieveTheUpdatedItems => "retrieve the updated items", - arroy::MainStep::RetrievingTheTreeAndItemNodes => "retrieving the tree and item nodes", - arroy::MainStep::UpdatingTheTrees => "updating the trees", - arroy::MainStep::CreateNewTrees => "create new trees", - arroy::MainStep::WritingNodesToDatabase => "writing nodes to database", - arroy::MainStep::DeleteExtraneousTrees => "delete extraneous trees", - arroy::MainStep::WriteTheMetadata => "write the metadata", - } - .into() - } +// impl Step for hannoy::MainStep { +// fn name(&self) -> Cow<'static, str> { +// match self { +// hannoy::MainStep::PreProcessingTheItems => "pre processing the items", +// hannoy::MainStep::WritingTheDescendantsAndMetadata => { +// "writing the descendants and metadata" +// } +// hannoy::MainStep::RetrieveTheUpdatedItems => "retrieve the updated items", +// hannoy::MainStep::RetrievingTheTreeAndItemNodes => "retrieving the tree and item nodes", +// hannoy::MainStep::UpdatingTheTrees => "updating the trees", +// hannoy::MainStep::CreateNewTrees => "create new trees", +// hannoy::MainStep::WritingNodesToDatabase => "writing nodes to database", +// hannoy::MainStep::DeleteExtraneousTrees => "delete extraneous trees", +// hannoy::MainStep::WriteTheMetadata => "write the metadata", +// } +// .into() +// } - fn current(&self) -> u32 { - *self as u32 - } +// fn current(&self) -> u32 { +// *self as u32 +// } - fn total(&self) -> u32 { - Self::CARDINALITY as u32 - } -} +// fn total(&self) -> u32 { +// Self::CARDINALITY as u32 +// } +// } -impl Step for arroy::SubStep { - fn name(&self) -> Cow<'static, str> { - self.unit.into() - } +// impl Step for hannoy::SubStep { +// fn name(&self) -> Cow<'static, str> { +// self.unit.into() +// } - fn current(&self) -> u32 { - self.current.load(Ordering::Relaxed) - } +// fn current(&self) -> u32 { +// self.current.load(Ordering::Relaxed) +// } - fn total(&self) -> u32 { - self.max - } -} +// fn total(&self) -> u32 { +// self.max +// } +// } diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index 2c201e899..71f7faa48 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -6,7 +6,7 @@ use roaring::RoaringBitmap; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use super::VectorStoreStats; use crate::score_details::{self, ScoreDetails}; -use crate::vector::{ArroyWrapper, DistributionShift, Embedder}; +use crate::vector::{DistributionShift, Embedder, HannoyWrapper}; use crate::{DocumentId, Result, SearchContext, SearchLogger}; pub struct VectorSort { @@ -56,7 +56,8 @@ impl VectorSort { let target = &self.target; let before = Instant::now(); - let reader = ArroyWrapper::new(ctx.index.vector_arroy, self.embedder_index, self.quantized); + let reader = + HannoyWrapper::new(ctx.index.vector_hannoy, self.embedder_index, self.quantized); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 2235f6436..2e70958c0 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use roaring::RoaringBitmap; use crate::score_details::{self, ScoreDetails}; -use crate::vector::{ArroyWrapper, Embedder}; +use crate::vector::{Embedder, HannoyWrapper}; use crate::{filtered_universe, DocumentId, Filter, Index, Result, SearchResult}; pub struct Similar<'a> { @@ -72,7 +72,7 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let reader = ArroyWrapper::new(self.index.vector_arroy, embedder_index, self.quantized); + let reader = HannoyWrapper::new(self.index.vector_hannoy, embedder_index, self.quantized); let results = reader.nns_by_item( self.rtxn, self.id, diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index 84eeca7f9..164aed9a0 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -45,7 +45,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_arroy, + vector_hannoy, embedder_category_id: _, documents, } = self.index; @@ -88,7 +88,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { field_id_docid_facet_f64s.clear(self.wtxn)?; field_id_docid_facet_strings.clear(self.wtxn)?; // vector - vector_arroy.clear(self.wtxn)?; + vector_hannoy.clear(self.wtxn)?; documents.clear(self.wtxn)?; diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 099879382..ec1aac32a 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -39,7 +39,7 @@ use crate::update::{ IndexerConfig, UpdateIndexingStep, WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst, }; use crate::vector::db::EmbedderInfo; -use crate::vector::{ArroyWrapper, RuntimeEmbedders}; +use crate::vector::{HannoyWrapper, RuntimeEmbedders}; use crate::{CboRoaringBitmapCodec, Index, Result, UserError}; static MERGED_DATABASE_COUNT: usize = 7; @@ -494,7 +494,7 @@ where }, )?; let reader = - ArroyWrapper::new(self.index.vector_arroy, index, action.was_quantized); + HannoyWrapper::new(self.index.vector_hannoy, index, action.was_quantized); let Some(dim) = reader.dimensions(self.wtxn)? else { continue; }; @@ -504,7 +504,7 @@ where for (embedder_name, dimension) in dimension { let wtxn = &mut *self.wtxn; - let vector_arroy = self.index.vector_arroy; + let vector_hannoy = self.index.vector_hannoy; let cancel = &self.should_abort; let embedder_index = @@ -523,7 +523,7 @@ where let is_quantizing = embedder_config.is_some_and(|action| action.is_being_quantized); pool.install(|| { - let mut writer = ArroyWrapper::new(vector_arroy, embedder_index, was_quantized); + let mut writer = HannoyWrapper::new(vector_hannoy, embedder_index, was_quantized); writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index e07483aff..fcb5b00d1 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -32,7 +32,7 @@ use crate::update::settings::{InnerIndexSettings, InnerIndexSettingsDiff}; use crate::update::{AvailableIds, UpdateIndexingStep}; use crate::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use crate::vector::settings::{RemoveFragments, WriteBackToDocuments}; -use crate::vector::ArroyWrapper; +use crate::vector::HannoyWrapper; use crate::{FieldDistribution, FieldId, FieldIdMapMissingEntry, Index, Result}; pub struct TransformOutput { @@ -834,15 +834,15 @@ impl<'a, 'i> Transform<'a, 'i> { None }; - let readers: BTreeMap<&str, (ArroyWrapper, &RoaringBitmap)> = settings_diff + let readers: BTreeMap<&str, (HannoyWrapper, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() .filter_map(|(name, action)| { if let Some(WriteBackToDocuments { embedder_id, user_provided }) = action.write_back() { - let reader = ArroyWrapper::new( - self.index.vector_arroy, + let reader = HannoyWrapper::new( + self.index.vector_hannoy, *embedder_id, action.was_quantized, ); @@ -884,7 +884,7 @@ impl<'a, 'i> Transform<'a, 'i> { let injected_vectors: std::result::Result< serde_json::Map, - arroy::Error, + hannoy::Error, > = readers .iter() .filter_map(|(name, (reader, user_provided))| { @@ -949,9 +949,9 @@ impl<'a, 'i> Transform<'a, 'i> { else { continue; }; - let arroy = - ArroyWrapper::new(self.index.vector_arroy, infos.embedder_id, was_quantized); - let Some(dimensions) = arroy.dimensions(wtxn)? else { + let hannoy = + HannoyWrapper::new(self.index.vector_hannoy, infos.embedder_id, was_quantized); + let Some(dimensions) = hannoy.dimensions(wtxn)? else { continue; }; for fragment_id in fragment_ids { @@ -959,17 +959,17 @@ impl<'a, 'i> Transform<'a, 'i> { if infos.embedding_status.user_provided_docids().is_empty() { // no user provided: clear store - arroy.clear_store(wtxn, *fragment_id, dimensions)?; + hannoy.clear_store(wtxn, *fragment_id, dimensions)?; continue; } // some user provided, remove only the ids that are not user provided - let to_delete = arroy.items_in_store(wtxn, *fragment_id, |items| { + let to_delete = hannoy.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() })?; for to_delete in to_delete { - arroy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; + hannoy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; } } } diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index c93e3e0f7..b0590eab4 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -27,7 +27,7 @@ use crate::update::index_documents::helpers::{ }; use crate::update::settings::InnerIndexSettingsDiff; use crate::vector::db::{EmbeddingStatusDelta, IndexEmbeddingConfig}; -use crate::vector::ArroyWrapper; +use crate::vector::HannoyWrapper; use crate::{ lat_lng_to_xyz, CboRoaringBitmapCodec, DocumentId, FieldId, GeoPoint, Index, InternalError, Result, SerializationError, U8StrStrCodec, @@ -677,7 +677,8 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .is_some_and(|conf| conf.is_quantized); // FIXME: allow customizing distance - let writer = ArroyWrapper::new(index.vector_arroy, infos.embedder_id, binary_quantized); + let writer = + HannoyWrapper::new(index.vector_hannoy, infos.embedder_id, binary_quantized); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); diff --git a/crates/milli/src/update/new/channel.rs b/crates/milli/src/update/new/channel.rs index aec192ace..86843795b 100644 --- a/crates/milli/src/update/new/channel.rs +++ b/crates/milli/src/update/new/channel.rs @@ -255,9 +255,9 @@ impl<'a> From> for FrameWithHeader<'a> { #[repr(u8)] pub enum EntryHeader { DbOperation(DbOperation), - ArroyDeleteVector(ArroyDeleteVector), - ArroySetVectors(ArroySetVectors), - ArroySetVector(ArroySetVector), + HannoyDeleteVector(HannoyDeleteVector), + HannoySetVectors(HannoySetVectors), + HannoySetVector(HannoySetVector), } impl EntryHeader { @@ -268,9 +268,9 @@ impl EntryHeader { const fn variant_id(&self) -> u8 { match self { EntryHeader::DbOperation(_) => 0, - EntryHeader::ArroyDeleteVector(_) => 1, - EntryHeader::ArroySetVectors(_) => 2, - EntryHeader::ArroySetVector(_) => 3, + EntryHeader::HannoyDeleteVector(_) => 1, + EntryHeader::HannoySetVectors(_) => 2, + EntryHeader::HannoySetVector(_) => 3, } } @@ -286,26 +286,26 @@ impl EntryHeader { } const fn total_delete_vector_size() -> usize { - Self::variant_size() + mem::size_of::() + Self::variant_size() + mem::size_of::() } /// The `dimensions` corresponds to the number of `f32` in the embedding. fn total_set_vectors_size(count: usize, dimensions: usize) -> usize { let embedding_size = dimensions * mem::size_of::(); - Self::variant_size() + mem::size_of::() + embedding_size * count + Self::variant_size() + mem::size_of::() + embedding_size * count } fn total_set_vector_size(dimensions: usize) -> usize { let embedding_size = dimensions * mem::size_of::(); - Self::variant_size() + mem::size_of::() + embedding_size + Self::variant_size() + mem::size_of::() + embedding_size } fn header_size(&self) -> usize { let payload_size = match self { EntryHeader::DbOperation(op) => mem::size_of_val(op), - EntryHeader::ArroyDeleteVector(adv) => mem::size_of_val(adv), - EntryHeader::ArroySetVectors(asvs) => mem::size_of_val(asvs), - EntryHeader::ArroySetVector(asv) => mem::size_of_val(asv), + EntryHeader::HannoyDeleteVector(adv) => mem::size_of_val(adv), + EntryHeader::HannoySetVectors(asvs) => mem::size_of_val(asvs), + EntryHeader::HannoySetVector(asv) => mem::size_of_val(asv), }; Self::variant_size() + payload_size } @@ -319,19 +319,19 @@ impl EntryHeader { EntryHeader::DbOperation(header) } 1 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::ArroyDeleteVector(header) + EntryHeader::HannoyDeleteVector(header) } 2 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::ArroySetVectors(header) + EntryHeader::HannoySetVectors(header) } 3 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::ArroySetVector(header) + EntryHeader::HannoySetVector(header) } id => panic!("invalid variant id: {id}"), } @@ -341,9 +341,9 @@ impl EntryHeader { let (first, remaining) = header_bytes.split_first_mut().unwrap(); let payload_bytes = match self { EntryHeader::DbOperation(op) => bytemuck::bytes_of(op), - EntryHeader::ArroyDeleteVector(adv) => bytemuck::bytes_of(adv), - EntryHeader::ArroySetVectors(asvs) => bytemuck::bytes_of(asvs), - EntryHeader::ArroySetVector(asv) => bytemuck::bytes_of(asv), + EntryHeader::HannoyDeleteVector(adv) => bytemuck::bytes_of(adv), + EntryHeader::HannoySetVectors(asvs) => bytemuck::bytes_of(asvs), + EntryHeader::HannoySetVector(asv) => bytemuck::bytes_of(asv), }; *first = self.variant_id(); remaining.copy_from_slice(payload_bytes); @@ -378,7 +378,7 @@ impl DbOperation { #[derive(Debug, Clone, Copy, NoUninit, CheckedBitPattern)] #[repr(transparent)] -pub struct ArroyDeleteVector { +pub struct HannoyDeleteVector { pub docid: DocumentId, } @@ -386,13 +386,13 @@ pub struct ArroyDeleteVector { #[repr(C)] /// The embeddings are in the remaining space and represents /// non-aligned [f32] each with dimensions f32s. -pub struct ArroySetVectors { +pub struct HannoySetVectors { pub docid: DocumentId, pub embedder_id: u8, _padding: [u8; 3], } -impl ArroySetVectors { +impl HannoySetVectors { fn embeddings_bytes<'a>(frame: &'a FrameGrantR<'_>) -> &'a [u8] { let skip = EntryHeader::variant_size() + mem::size_of::(); &frame[skip..] @@ -416,14 +416,14 @@ impl ArroySetVectors { #[repr(C)] /// The embeddings are in the remaining space and represents /// non-aligned [f32] each with dimensions f32s. -pub struct ArroySetVector { +pub struct HannoySetVector { pub docid: DocumentId, pub embedder_id: u8, pub extractor_id: u8, _padding: [u8; 2], } -impl ArroySetVector { +impl HannoySetVector { fn embeddings_bytes<'a>(frame: &'a FrameGrantR<'_>) -> &'a [u8] { let skip = EntryHeader::variant_size() + mem::size_of::(); &frame[skip..] @@ -553,7 +553,7 @@ impl<'b> ExtractorBbqueueSender<'b> { let refcell = self.producers.get().unwrap(); let mut producer = refcell.0.borrow_mut_or_yield(); - let payload_header = EntryHeader::ArroyDeleteVector(ArroyDeleteVector { docid }); + let payload_header = EntryHeader::HannoyDeleteVector(HannoyDeleteVector { docid }); let total_length = EntryHeader::total_delete_vector_size(); if total_length > max_grant { panic!("The entry is larger ({total_length} bytes) than the BBQueue max grant ({max_grant} bytes)"); @@ -589,8 +589,8 @@ impl<'b> ExtractorBbqueueSender<'b> { // to zero to allocate no extra space at all let dimensions = embeddings.first().map_or(0, |emb| emb.len()); - let arroy_set_vector = ArroySetVectors { docid, embedder_id, _padding: [0; 3] }; - let payload_header = EntryHeader::ArroySetVectors(arroy_set_vector); + let hannoy_set_vector = HannoySetVectors { docid, embedder_id, _padding: [0; 3] }; + let payload_header = EntryHeader::HannoySetVectors(hannoy_set_vector); let total_length = EntryHeader::total_set_vectors_size(embeddings.len(), dimensions); if total_length > max_grant { let mut value_file = tempfile::tempfile().map(BufWriter::new)?; @@ -650,9 +650,9 @@ impl<'b> ExtractorBbqueueSender<'b> { // to zero to allocate no extra space at all let dimensions = embedding.as_ref().map_or(0, |emb| emb.len()); - let arroy_set_vector = - ArroySetVector { docid, embedder_id, extractor_id, _padding: [0; 2] }; - let payload_header = EntryHeader::ArroySetVector(arroy_set_vector); + let hannoy_set_vector = + HannoySetVector { docid, embedder_id, extractor_id, _padding: [0; 2] }; + let payload_header = EntryHeader::HannoySetVector(hannoy_set_vector); let total_length = EntryHeader::total_set_vector_size(dimensions); if total_length > max_grant { let mut value_file = tempfile::tempfile().map(BufWriter::new)?; diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index e18337623..35db7b9fc 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -24,7 +24,7 @@ use crate::progress::{EmbedderStats, Progress}; use crate::update::settings::SettingsDelta; use crate::update::GrenadParameters; use crate::vector::settings::{EmbedderAction, RemoveFragments, WriteBackToDocuments}; -use crate::vector::{ArroyWrapper, Embedder, RuntimeEmbedders}; +use crate::vector::{Embedder, HannoyWrapper, RuntimeEmbedders}; use crate::{FieldsIdsMap, GlobalFieldsIdsMap, Index, InternalError, Result, ThreadPoolNoAbort}; pub(crate) mod de; @@ -67,7 +67,7 @@ where let mut bbbuffers = Vec::new(); let finished_extraction = AtomicBool::new(false); - let arroy_memory = grenad_parameters.max_memory; + let hannoy_memory = grenad_parameters.max_memory; let (grenad_parameters, total_bbbuffer_capacity) = indexer_memory_settings(pool.current_num_threads(), grenad_parameters); @@ -130,8 +130,8 @@ where let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map); - let vector_arroy = index.vector_arroy; - let arroy_writers: Result> = embedders + let vector_arroy = index.vector_hannoy; + let hannoy_writers: Result> = embedders .inner_as_ref() .iter() .map(|(embedder_name, runtime)| { @@ -144,7 +144,7 @@ where })?; let dimensions = runtime.embedder.dimensions(); - let writer = ArroyWrapper::new(vector_arroy, embedder_index, runtime.is_quantized); + let writer = HannoyWrapper::new(vector_arroy, embedder_index, runtime.is_quantized); Ok(( embedder_index, @@ -153,10 +153,10 @@ where }) .collect(); - let mut arroy_writers = arroy_writers?; + let mut hannoy_writers = hannoy_writers?; let congestion = - write_to_db(writer_receiver, finished_extraction, index, wtxn, &arroy_writers)?; + write_to_db(writer_receiver, finished_extraction, index, wtxn, &hannoy_writers)?; indexing_context.progress.update_progress(IndexingStep::WaitingForExtractors); @@ -170,8 +170,8 @@ where wtxn, indexing_context.progress, index_embeddings, - arroy_memory, - &mut arroy_writers, + hannoy_memory, + &mut hannoy_writers, None, &indexing_context.must_stop_processing, ) @@ -227,7 +227,7 @@ where let mut bbbuffers = Vec::new(); let finished_extraction = AtomicBool::new(false); - let arroy_memory = grenad_parameters.max_memory; + let hannoy_memory = grenad_parameters.max_memory; let (grenad_parameters, total_bbbuffer_capacity) = indexer_memory_settings(pool.current_num_threads(), grenad_parameters); @@ -284,7 +284,7 @@ where let new_embedders = settings_delta.new_embedders(); let embedder_actions = settings_delta.embedder_actions(); let index_embedder_category_ids = settings_delta.new_embedder_category_id(); - let mut arroy_writers = arroy_writers_from_embedder_actions( + let mut hannoy_writers = hannoy_writers_from_embedder_actions( index, embedder_actions, new_embedders, @@ -292,7 +292,7 @@ where )?; let congestion = - write_to_db(writer_receiver, finished_extraction, index, wtxn, &arroy_writers)?; + write_to_db(writer_receiver, finished_extraction, index, wtxn, &hannoy_writers)?; indexing_context.progress.update_progress(IndexingStep::WaitingForExtractors); @@ -306,8 +306,8 @@ where wtxn, indexing_context.progress, index_embeddings, - arroy_memory, - &mut arroy_writers, + hannoy_memory, + &mut hannoy_writers, Some(embedder_actions), &indexing_context.must_stop_processing, ) @@ -337,13 +337,13 @@ where Ok(congestion) } -fn arroy_writers_from_embedder_actions<'indexer>( +fn hannoy_writers_from_embedder_actions<'indexer>( index: &Index, embedder_actions: &'indexer BTreeMap, embedders: &'indexer RuntimeEmbedders, index_embedder_category_ids: &'indexer std::collections::HashMap, -) -> Result> { - let vector_arroy = index.vector_arroy; +) -> Result> { + let vector_arroy = index.vector_hannoy; embedders .inner_as_ref() @@ -362,7 +362,7 @@ fn arroy_writers_from_embedder_actions<'indexer>( ))); }; let writer = - ArroyWrapper::new(vector_arroy, embedder_category_id, action.was_quantized); + HannoyWrapper::new(vector_arroy, embedder_category_id, action.was_quantized); let dimensions = runtime.embedder.dimensions(); Some(Ok(( embedder_category_id, @@ -385,7 +385,7 @@ where let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; }; - let reader = ArroyWrapper::new(index.vector_arroy, *embedder_id, action.was_quantized); + let reader = HannoyWrapper::new(index.vector_hannoy, *embedder_id, action.was_quantized); let Some(dimensions) = reader.dimensions(wtxn)? else { continue; }; @@ -401,7 +401,7 @@ where let Some(infos) = index.embedding_configs().embedder_info(wtxn, embedder_name)? else { continue; }; - let arroy = ArroyWrapper::new(index.vector_arroy, infos.embedder_id, was_quantized); + let arroy = HannoyWrapper::new(index.vector_hannoy, infos.embedder_id, was_quantized); let Some(dimensions) = arroy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/new/indexer/write.rs b/crates/milli/src/update/new/indexer/write.rs index b8e3685f8..4be916c02 100644 --- a/crates/milli/src/update/new/indexer/write.rs +++ b/crates/milli/src/update/new/indexer/write.rs @@ -15,7 +15,7 @@ use crate::progress::Progress; use crate::update::settings::InnerIndexSettings; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::settings::EmbedderAction; -use crate::vector::{ArroyWrapper, Embedder, Embeddings, RuntimeEmbedders}; +use crate::vector::{Embedder, Embeddings, HannoyWrapper, RuntimeEmbedders}; use crate::{Error, Index, InternalError, Result, UserError}; pub fn write_to_db( @@ -23,9 +23,9 @@ pub fn write_to_db( finished_extraction: &AtomicBool, index: &Index, wtxn: &mut RwTxn<'_>, - arroy_writers: &HashMap, + hannoy_writers: &HashMap, ) -> Result { - // Used by by the ArroySetVector to copy the embedding into an + // Used by by the HannoySetVector to copy the embedding into an // aligned memory area, required by arroy to accept a new vector. let mut aligned_embedding = Vec::new(); let span = tracing::trace_span!(target: "indexing::write_db", "all"); @@ -56,7 +56,7 @@ pub fn write_to_db( ReceiverAction::LargeVectors(large_vectors) => { let LargeVectors { docid, embedder_id, .. } = large_vectors; let (_, _, writer, dimensions) = - arroy_writers.get(&embedder_id).expect("requested a missing embedder"); + hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); let mut embeddings = Embeddings::new(*dimensions); for embedding in large_vectors.read_embeddings(*dimensions) { embeddings.push(embedding.to_vec()).unwrap(); @@ -68,7 +68,7 @@ pub fn write_to_db( large_vector @ LargeVector { docid, embedder_id, extractor_id, .. }, ) => { let (_, _, writer, dimensions) = - arroy_writers.get(&embedder_id).expect("requested a missing embedder"); + hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); let embedding = large_vector.read_embedding(*dimensions); writer.add_item_in_store(wtxn, docid, extractor_id, embedding)?; } @@ -80,12 +80,12 @@ pub fn write_to_db( &mut writer_receiver, index, wtxn, - arroy_writers, + hannoy_writers, &mut aligned_embedding, )?; } - write_from_bbqueue(&mut writer_receiver, index, wtxn, arroy_writers, &mut aligned_embedding)?; + write_from_bbqueue(&mut writer_receiver, index, wtxn, hannoy_writers, &mut aligned_embedding)?; Ok(ChannelCongestion { attempts: writer_receiver.sent_messages_attempts(), @@ -115,8 +115,8 @@ pub fn build_vectors( wtxn: &mut RwTxn<'_>, progress: &Progress, index_embeddings: Vec, - arroy_memory: Option, - arroy_writers: &mut HashMap, + hannoy_memory: Option, + hannoy_writers: &mut HashMap, embeder_actions: Option<&BTreeMap>, must_stop_processing: &MSP, ) -> Result<()> @@ -129,7 +129,7 @@ where let seed = rand::random(); let mut rng = rand::rngs::StdRng::seed_from_u64(seed); - for (_index, (embedder_name, _embedder, writer, dimensions)) in arroy_writers { + for (_index, (embedder_name, _embedder, writer, dimensions)) in hannoy_writers { let dimensions = *dimensions; let is_being_quantized = embeder_actions .and_then(|actions| actions.get(*embedder_name).map(|action| action.is_being_quantized)) @@ -140,7 +140,7 @@ where &mut rng, dimensions, is_being_quantized, - arroy_memory, + hannoy_memory, must_stop_processing, )?; } @@ -181,7 +181,7 @@ pub fn write_from_bbqueue( writer_receiver: &mut WriterBbqueueReceiver<'_>, index: &Index, wtxn: &mut RwTxn<'_>, - arroy_writers: &HashMap, + hannoy_writers: &HashMap, aligned_embedding: &mut Vec, ) -> crate::Result<()> { while let Some(frame_with_header) = writer_receiver.recv_frame() { @@ -221,17 +221,17 @@ pub fn write_from_bbqueue( }, } } - EntryHeader::ArroyDeleteVector(ArroyDeleteVector { docid }) => { - for (_index, (_name, _embedder, writer, dimensions)) in arroy_writers { + EntryHeader::HannoyDeleteVector(HannoyDeleteVector { docid }) => { + for (_index, (_name, _embedder, writer, dimensions)) in hannoy_writers { let dimensions = *dimensions; writer.del_items(wtxn, dimensions, docid)?; } } - EntryHeader::ArroySetVectors(asvs) => { - let ArroySetVectors { docid, embedder_id, .. } = asvs; + EntryHeader::HannoySetVectors(asvs) => { + let HannoySetVectors { docid, embedder_id, .. } = asvs; let frame = frame_with_header.frame(); let (_, _, writer, dimensions) = - arroy_writers.get(&embedder_id).expect("requested a missing embedder"); + hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); let mut embeddings = Embeddings::new(*dimensions); let all_embeddings = asvs.read_all_embeddings_into_vec(frame, aligned_embedding); writer.del_items(wtxn, *dimensions, docid)?; @@ -245,12 +245,12 @@ pub fn write_from_bbqueue( writer.add_items(wtxn, docid, &embeddings)?; } } - EntryHeader::ArroySetVector( - asv @ ArroySetVector { docid, embedder_id, extractor_id, .. }, + EntryHeader::HannoySetVector( + asv @ HannoySetVector { docid, embedder_id, extractor_id, .. }, ) => { let frame = frame_with_header.frame(); let (_, _, writer, dimensions) = - arroy_writers.get(&embedder_id).expect("requested a missing embedder"); + hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); let embedding = asv.read_all_embeddings_into_vec(frame, aligned_embedding); if embedding.is_empty() { diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index b59984248..64e1377ad 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -14,7 +14,7 @@ use crate::constants::RESERVED_VECTORS_FIELD_NAME; use crate::documents::FieldIdMapper; use crate::vector::db::{EmbeddingStatus, IndexEmbeddingConfig}; use crate::vector::parsed_vectors::{RawVectors, RawVectorsError, VectorOrArrayOfVectors}; -use crate::vector::{ArroyWrapper, Embedding, RuntimeEmbedders}; +use crate::vector::{Embedding, HannoyWrapper, RuntimeEmbedders}; use crate::{DocumentId, Index, InternalError, Result, UserError}; #[derive(Serialize)] @@ -121,7 +121,7 @@ impl<'t> VectorDocumentFromDb<'t> { status: &EmbeddingStatus, ) -> Result> { let reader = - ArroyWrapper::new(self.index.vector_arroy, embedder_id, config.config.quantized()); + HannoyWrapper::new(self.index.vector_hannoy, embedder_id, config.config.quantized()); let vectors = reader.item_vectors(self.rtxn, self.docid)?; Ok(VectorEntry { @@ -149,7 +149,7 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> { name, entry_from_raw_value(value, false).map_err(|_| { InternalError::Serialization(crate::SerializationError::Decoding { - db_name: Some(crate::index::db_name::VECTOR_ARROY), + db_name: Some(crate::index::db_name::VECTOR_HANNOY), }) })?, )) @@ -167,7 +167,7 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> { Some(embedding_from_doc) => { Some(entry_from_raw_value(embedding_from_doc, false).map_err(|_| { InternalError::Serialization(crate::SerializationError::Decoding { - db_name: Some(crate::index::db_name::VECTOR_ARROY), + db_name: Some(crate::index::db_name::VECTOR_HANNOY), }) })?) } diff --git a/crates/milli/src/update/upgrade/v1_14.rs b/crates/milli/src/update/upgrade/v1_14.rs index 039734b75..68ff9f8cd 100644 --- a/crates/milli/src/update/upgrade/v1_14.rs +++ b/crates/milli/src/update/upgrade/v1_14.rs @@ -1,4 +1,4 @@ -use arroy::distances::Cosine; +use hannoy::distances::Cosine; use heed::RwTxn; use super::UpgradeIndex; @@ -25,12 +25,13 @@ impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 { progress.update_progress(VectorStore::UpdateInternalVersions); let rtxn = index.read_txn()?; - arroy::upgrade::from_0_5_to_0_6::( - &rtxn, - index.vector_arroy.remap_data_type(), - wtxn, - index.vector_arroy.remap_data_type(), - )?; + // hannoy::upgrade::from_0_5_to_0_6::( + // &rtxn, + // index.vector_hannoy.remap_data_type(), + // wtxn, + // index.vector_hannoy.remap_data_type(), + // )?; + unimplemented!("upgrade hannoy"); Ok(false) } diff --git a/crates/milli/src/vector/composite.rs b/crates/milli/src/vector/composite.rs index 2e31da094..539e92ba8 100644 --- a/crates/milli/src/vector/composite.rs +++ b/crates/milli/src/vector/composite.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use arroy::Distance; +use hannoy::Distance; use super::error::CompositeEmbedderContainsHuggingFace; use super::{ @@ -324,19 +324,18 @@ fn check_similarity( } for (left, right) in left.into_iter().zip(right) { - let left = arroy::internals::UnalignedVector::from_slice(&left); - let right = arroy::internals::UnalignedVector::from_slice(&right); - let left = arroy::internals::Leaf { - header: arroy::distances::Cosine::new_header(&left), + let left = hannoy::internals::UnalignedVector::from_slice(&left); + let right = hannoy::internals::UnalignedVector::from_slice(&right); + let left = hannoy::internals::Item { + header: hannoy::distances::Cosine::new_header(&left), vector: left, }; - let right = arroy::internals::Leaf { - header: arroy::distances::Cosine::new_header(&right), + let right = hannoy::internals::Item { + header: hannoy::distances::Cosine::new_header(&right), vector: right, }; - let distance = arroy::distances::Cosine::built_distance(&left, &right); - + let distance = hannoy::distances::Cosine::distance(&left, &right); if distance > super::MAX_COMPOSITE_DISTANCE { return Err(NewEmbedderError::composite_embedding_value_mismatch(distance, hint)); } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 1f07f6c4f..bd5c651a6 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -3,9 +3,9 @@ use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; use std::time::Instant; -use arroy::distances::{BinaryQuantizedCosine, Cosine}; -use arroy::ItemId; use deserr::{DeserializeError, Deserr}; +use hannoy::distances::{BinaryQuantizedCosine, Cosine}; +use hannoy::ItemId; use heed::{RoTxn, RwTxn, Unspecified}; use ordered_float::OrderedFloat; use roaring::RoaringBitmap; @@ -41,15 +41,15 @@ pub type Embedding = Vec; pub const REQUEST_PARALLELISM: usize = 40; pub const MAX_COMPOSITE_DISTANCE: f32 = 0.01; -pub struct ArroyWrapper { +pub struct HannoyWrapper { quantized: bool, embedder_index: u8, - database: arroy::Database, + database: hannoy::Database, } -impl ArroyWrapper { +impl HannoyWrapper { pub fn new( - database: arroy::Database, + database: hannoy::Database, embedder_index: u8, quantized: bool, ) -> Self { @@ -60,19 +60,19 @@ impl ArroyWrapper { self.embedder_index } - fn readers<'a, D: arroy::Distance>( + fn readers<'a, D: hannoy::Distance>( &'a self, rtxn: &'a RoTxn<'a>, - db: arroy::Database, - ) -> impl Iterator, arroy::Error>> + 'a { - arroy_store_range_for_embedder(self.embedder_index).filter_map(move |index| { - match arroy::Reader::open(rtxn, index, db) { + db: hannoy::Database, + ) -> impl Iterator, hannoy::Error>> + 'a { + hannoy_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match hannoy::Reader::open(rtxn, index, db) { Ok(reader) => match reader.is_empty(rtxn) { Ok(false) => Some(Ok(reader)), Ok(true) => None, Err(e) => Some(Err(e)), }, - Err(arroy::Error::MissingMetadata(_)) => None, + Err(hannoy::Error::MissingMetadata(_)) => None, Err(e) => Some(Err(e)), } }) @@ -86,7 +86,7 @@ impl ArroyWrapper { rtxn: &RoTxn, store_id: u8, with_items: F, - ) -> Result + ) -> Result where F: FnOnce(&RoaringBitmap) -> O, { @@ -97,26 +97,26 @@ impl ArroyWrapper { } } - fn _items_in_store( + fn _items_in_store( &self, rtxn: &RoTxn, - db: arroy::Database, + db: hannoy::Database, store_id: u8, with_items: F, - ) -> Result + ) -> Result where F: FnOnce(&RoaringBitmap) -> O, { - let index = arroy_store_for_embedder(self.embedder_index, store_id); - let reader = arroy::Reader::open(rtxn, index, db); + let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let reader = hannoy::Reader::open(rtxn, index, db); match reader { Ok(reader) => Ok(with_items(reader.item_ids())), - Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(hannoy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), Err(err) => Err(err), } } - pub fn dimensions(&self, rtxn: &RoTxn) -> Result, arroy::Error> { + pub fn dimensions(&self, rtxn: &RoTxn) -> Result, hannoy::Error> { if self.quantized { Ok(self .readers(rtxn, self.quantized_db()) @@ -140,39 +140,41 @@ impl ArroyWrapper { rng: &mut R, dimension: usize, quantizing: bool, - arroy_memory: Option, + hannoy_memory: Option, cancel: &(impl Fn() -> bool + Sync + Send), - ) -> Result<(), arroy::Error> { - for index in arroy_store_range_for_embedder(self.embedder_index) { + ) -> Result<(), hannoy::Error> { + for index in hannoy_store_range_for_embedder(self.embedder_index) { if self.quantized { - let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.need_build(wtxn)? { - writer.builder(rng).build(wtxn)? + writer.builder(rng).ef_construction(48).build::<16, 32>(wtxn)? } else if writer.is_empty(wtxn)? { continue; } } else { - let writer = arroy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); // If we are quantizing the databases, we can't know from meilisearch // if the db was empty but still contained the wrong metadata, thus we need // to quantize everything and can't stop early. Since this operation can // only happens once in the life of an embedder, it's not very performances // sensitive. if quantizing && !self.quantized { - let writer = writer.prepare_changing_distance::(wtxn)?; - writer - .builder(rng) - .available_memory(arroy_memory.unwrap_or(usize::MAX)) - .progress(|step| progress.update_progress_from_arroy(step)) - .cancel(cancel) - .build(wtxn)?; + // let writer = writer.prepare_changing_distance::(wtxn)?; + // writer + // .builder(rng) + // .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + // .progress(|step| progress.update_progress_from_hannoy(step)) + // .cancel(cancel) + // .build(wtxn)?; + unimplemented!("switching from quantized to non-quantized"); } else if writer.need_build(wtxn)? { writer .builder(rng) - .available_memory(arroy_memory.unwrap_or(usize::MAX)) - .progress(|step| progress.update_progress_from_arroy(step)) - .cancel(cancel) - .build(wtxn)?; + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + // .progress(|step| progress.update_progress_from_hannoy(step)) + // .cancel(cancel) + .ef_construction(48) + .build::<16, 32>(wtxn)?; } else if writer.is_empty(wtxn)? { continue; } @@ -188,18 +190,18 @@ impl ArroyWrapper { pub fn add_items( &self, wtxn: &mut RwTxn, - item_id: arroy::ItemId, + item_id: hannoy::ItemId, embeddings: &Embeddings, - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { let dimension = embeddings.dimension(); for (index, vector) in - arroy_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) + hannoy_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) { if self.quantized { - arroy::Writer::new(self.quantized_db(), index, dimension) + hannoy::Writer::new(self.quantized_db(), index, dimension) .add_item(wtxn, item_id, vector)? } else { - arroy::Writer::new(self.angular_db(), index, dimension) + hannoy::Writer::new(self.angular_db(), index, dimension) .add_item(wtxn, item_id, vector)? } } @@ -210,9 +212,9 @@ impl ArroyWrapper { pub fn add_item( &self, wtxn: &mut RwTxn, - item_id: arroy::ItemId, + item_id: hannoy::ItemId, vector: &[f32], - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { if self.quantized { self._add_item(wtxn, self.quantized_db(), item_id, vector) } else { @@ -220,17 +222,17 @@ impl ArroyWrapper { } } - fn _add_item( + fn _add_item( &self, wtxn: &mut RwTxn, - db: arroy::Database, - item_id: arroy::ItemId, + db: hannoy::Database, + item_id: hannoy::ItemId, vector: &[f32], - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { let dimension = vector.len(); - for index in arroy_store_range_for_embedder(self.embedder_index) { - let writer = arroy::Writer::new(db, index, dimension); + for index in hannoy_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); if !writer.contains_item(wtxn, item_id)? { writer.add_item(wtxn, item_id, vector)?; break; @@ -245,10 +247,10 @@ impl ArroyWrapper { pub fn add_item_in_store( &self, wtxn: &mut RwTxn, - item_id: arroy::ItemId, + item_id: hannoy::ItemId, store_id: u8, vector: &[f32], - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { if self.quantized { self._add_item_in_store(wtxn, self.quantized_db(), item_id, store_id, vector) } else { @@ -256,18 +258,18 @@ impl ArroyWrapper { } } - fn _add_item_in_store( + fn _add_item_in_store( &self, wtxn: &mut RwTxn, - db: arroy::Database, - item_id: arroy::ItemId, + db: hannoy::Database, + item_id: hannoy::ItemId, store_id: u8, vector: &[f32], - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { let dimension = vector.len(); - let index = arroy_store_for_embedder(self.embedder_index, store_id); - let writer = arroy::Writer::new(db, index, dimension); + let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimension); writer.add_item(wtxn, item_id, vector) } @@ -276,14 +278,14 @@ impl ArroyWrapper { &self, wtxn: &mut RwTxn, dimension: usize, - item_id: arroy::ItemId, - ) -> Result<(), arroy::Error> { - for index in arroy_store_range_for_embedder(self.embedder_index) { + item_id: hannoy::ItemId, + ) -> Result<(), hannoy::Error> { + for index in hannoy_store_range_for_embedder(self.embedder_index) { if self.quantized { - let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); writer.del_item(wtxn, item_id)?; } else { - let writer = arroy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); writer.del_item(wtxn, item_id)?; } } @@ -301,10 +303,10 @@ impl ArroyWrapper { pub fn del_item_in_store( &self, wtxn: &mut RwTxn, - item_id: arroy::ItemId, + item_id: hannoy::ItemId, store_id: u8, dimensions: usize, - ) -> Result { + ) -> Result { if self.quantized { self._del_item_in_store(wtxn, self.quantized_db(), item_id, store_id, dimensions) } else { @@ -312,16 +314,16 @@ impl ArroyWrapper { } } - fn _del_item_in_store( + fn _del_item_in_store( &self, wtxn: &mut RwTxn, - db: arroy::Database, - item_id: arroy::ItemId, + db: hannoy::Database, + item_id: hannoy::ItemId, store_id: u8, dimensions: usize, - ) -> Result { - let index = arroy_store_for_embedder(self.embedder_index, store_id); - let writer = arroy::Writer::new(db, index, dimensions); + ) -> Result { + let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); writer.del_item(wtxn, item_id) } @@ -335,7 +337,7 @@ impl ArroyWrapper { wtxn: &mut RwTxn, store_id: u8, dimensions: usize, - ) -> Result<(), arroy::Error> { + ) -> Result<(), hannoy::Error> { if self.quantized { self._clear_store(wtxn, self.quantized_db(), store_id, dimensions) } else { @@ -343,15 +345,15 @@ impl ArroyWrapper { } } - fn _clear_store( + fn _clear_store( &self, wtxn: &mut RwTxn, - db: arroy::Database, + db: hannoy::Database, store_id: u8, dimensions: usize, - ) -> Result<(), arroy::Error> { - let index = arroy_store_for_embedder(self.embedder_index, store_id); - let writer = arroy::Writer::new(db, index, dimensions); + ) -> Result<(), hannoy::Error> { + let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); writer.clear(wtxn) } @@ -359,9 +361,9 @@ impl ArroyWrapper { pub fn del_item( &self, wtxn: &mut RwTxn, - item_id: arroy::ItemId, + item_id: hannoy::ItemId, vector: &[f32], - ) -> Result { + ) -> Result { if self.quantized { self._del_item(wtxn, self.quantized_db(), item_id, vector) } else { @@ -369,37 +371,34 @@ impl ArroyWrapper { } } - fn _del_item( + fn _del_item( &self, wtxn: &mut RwTxn, - db: arroy::Database, - item_id: arroy::ItemId, + db: hannoy::Database, + item_id: hannoy::ItemId, vector: &[f32], - ) -> Result { + ) -> Result { let dimension = vector.len(); - for index in arroy_store_range_for_embedder(self.embedder_index) { - let writer = arroy::Writer::new(db, index, dimension); - let Some(candidate) = writer.item_vector(wtxn, item_id)? else { - continue; - }; - if candidate == vector { + for index in hannoy_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); + if writer.contains_item(wtxn, item_id)? { return writer.del_item(wtxn, item_id); } } Ok(false) } - pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), arroy::Error> { - for index in arroy_store_range_for_embedder(self.embedder_index) { + pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), hannoy::Error> { + for index in hannoy_store_range_for_embedder(self.embedder_index) { if self.quantized { - let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.is_empty(wtxn)? { continue; } writer.clear(wtxn)?; } else { - let writer = arroy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); if writer.is_empty(wtxn)? { continue; } @@ -413,17 +412,17 @@ impl ArroyWrapper { &self, rtxn: &RoTxn, dimension: usize, - item: arroy::ItemId, - ) -> Result { - for index in arroy_store_range_for_embedder(self.embedder_index) { + item: hannoy::ItemId, + ) -> Result { + for index in hannoy_store_range_for_embedder(self.embedder_index) { let contains = if self.quantized { - let writer = arroy::Writer::new(self.quantized_db(), index, dimension); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } writer.contains_item(rtxn, item)? } else { - let writer = arroy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } @@ -442,7 +441,7 @@ impl ArroyWrapper { item: ItemId, limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { + ) -> Result, hannoy::Error> { if self.quantized { self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) } else { @@ -450,19 +449,19 @@ impl ArroyWrapper { } } - fn _nns_by_item( + fn _nns_by_item( &self, rtxn: &RoTxn, - db: arroy::Database, + db: hannoy::Database, item: ItemId, limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { + ) -> Result, hannoy::Error> { let mut results = Vec::new(); for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit); + let mut searcher = reader.nns(limit, limit * 2); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; @@ -484,7 +483,7 @@ impl ArroyWrapper { vector: &[f32], limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { + ) -> Result, hannoy::Error> { if self.quantized { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { @@ -492,19 +491,19 @@ impl ArroyWrapper { } } - fn _nns_by_vector( + fn _nns_by_vector( &self, rtxn: &RoTxn, - db: arroy::Database, + db: hannoy::Database, vector: &[f32], limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { + ) -> Result, hannoy::Error> { let mut results = Vec::new(); for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit); + let mut searcher = reader.nns(limit, limit * 2); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; @@ -520,7 +519,7 @@ impl ArroyWrapper { Ok(results) } - pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> Result>, arroy::Error> { + pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> Result>, hannoy::Error> { let mut vectors = Vec::new(); if self.quantized { @@ -539,19 +538,19 @@ impl ArroyWrapper { Ok(vectors) } - fn angular_db(&self) -> arroy::Database { + fn angular_db(&self) -> hannoy::Database { self.database.remap_data_type() } - fn quantized_db(&self) -> arroy::Database { + fn quantized_db(&self) -> hannoy::Database { self.database.remap_data_type() } pub fn aggregate_stats( &self, rtxn: &RoTxn, - stats: &mut ArroyStats, - ) -> Result<(), arroy::Error> { + stats: &mut HannoyStats, + ) -> Result<(), hannoy::Error> { if self.quantized { for reader in self.readers(rtxn, self.quantized_db()) { let reader = reader?; @@ -573,10 +572,11 @@ impl ArroyWrapper { } #[derive(Debug, Default, Clone)] -pub struct ArroyStats { +pub struct HannoyStats { pub number_of_embeddings: u64, pub documents: RoaringBitmap, } + /// One or multiple embeddings stored consecutively in a flat vector. #[derive(Debug, PartialEq)] pub struct Embeddings { @@ -1221,11 +1221,11 @@ pub const fn is_cuda_enabled() -> bool { cfg!(feature = "cuda") } -fn arroy_store_range_for_embedder(embedder_id: u8) -> impl Iterator { - (0..=u8::MAX).map(move |store_id| arroy_store_for_embedder(embedder_id, store_id)) +fn hannoy_store_range_for_embedder(embedder_id: u8) -> impl Iterator { + (0..=u8::MAX).map(move |store_id| hannoy_store_for_embedder(embedder_id, store_id)) } -fn arroy_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { +fn hannoy_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { let embedder_id = (embedder_id as u16) << 8; embedder_id | (store_id as u16) } From a38a57acb631a5850ece7e7ed88142bce8835b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 29 Jul 2025 17:16:13 +0200 Subject: [PATCH 02/62] Use constants as the hannoy default parameters --- crates/milli/src/vector/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index bd5c651a6..512361029 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -41,6 +41,10 @@ pub type Embedding = Vec; pub const REQUEST_PARALLELISM: usize = 40; pub const MAX_COMPOSITE_DISTANCE: f32 = 0.01; +const HANNOY_EF_CONSTRUCTION: usize = 48; +const HANNOY_M: usize = 16; +const HANNOY_M0: usize = 32; + pub struct HannoyWrapper { quantized: bool, embedder_index: u8, @@ -147,7 +151,10 @@ impl HannoyWrapper { if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.need_build(wtxn)? { - writer.builder(rng).ef_construction(48).build::<16, 32>(wtxn)? + writer + .builder(rng) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)? } else if writer.is_empty(wtxn)? { continue; } @@ -173,8 +180,8 @@ impl HannoyWrapper { .available_memory(hannoy_memory.unwrap_or(usize::MAX)) // .progress(|step| progress.update_progress_from_hannoy(step)) // .cancel(cancel) - .ef_construction(48) - .build::<16, 32>(wtxn)?; + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; } else if writer.is_empty(wtxn)? { continue; } From f51f7832a740ce3ccdbf93d1f6252c068f5be693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 29 Jul 2025 17:30:02 +0200 Subject: [PATCH 03/62] Rename the ArroyWrapper/HannoyWrapper into VectorStore --- crates/milli/src/index.rs | 8 ++++---- crates/milli/src/search/new/vector_sort.rs | 5 ++--- crates/milli/src/search/similar.rs | 4 ++-- crates/milli/src/update/index_documents/mod.rs | 6 +++--- crates/milli/src/update/index_documents/transform.rs | 8 ++++---- .../milli/src/update/index_documents/typed_chunk.rs | 5 ++--- crates/milli/src/update/new/indexer/mod.rs | 12 ++++++------ crates/milli/src/update/new/indexer/write.rs | 8 ++++---- crates/milli/src/update/new/vector_document.rs | 4 ++-- crates/milli/src/vector/mod.rs | 4 ++-- 10 files changed, 31 insertions(+), 33 deletions(-) diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 4c4712c63..c4651b5a9 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -31,7 +31,7 @@ use crate::prompt::PromptData; use crate::proximity::ProximityPrecision; use crate::update::new::StdResult; use crate::vector::db::IndexEmbeddingConfigs; -use crate::vector::{Embedding, HannoyStats, HannoyWrapper}; +use crate::vector::{Embedding, HannoyStats, VectorStore}; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldIdWordCountCodec, @@ -237,7 +237,7 @@ impl Index { // vector stuff let embedder_category_id = env.create_database(&mut wtxn, Some(VECTOR_EMBEDDER_CATEGORY_ID))?; - let vector_hannoy = env.create_database(&mut wtxn, Some(VECTOR_HANNOY))?; + let vector_hannoy = env.create_database(&mut wtxn, Some(VECTOR_STORE))?; let documents = env.create_database(&mut wtxn, Some(DOCUMENTS))?; @@ -1772,7 +1772,7 @@ impl Index { for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); let has_fragments = config.config.embedder_options.has_fragments(); - let reader = HannoyWrapper::new( + let reader = VectorStore::new( self.vector_hannoy, embedder_info.embedder_id, config.config.quantized(), @@ -1798,7 +1798,7 @@ impl Index { for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); let reader = - HannoyWrapper::new(self.vector_hannoy, embedder_id, config.config.quantized()); + VectorStore::new(self.vector_hannoy, embedder_id, config.config.quantized()); reader.aggregate_stats(rtxn, &mut stats)?; } Ok(stats) diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index 71f7faa48..ce755c57d 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -6,7 +6,7 @@ use roaring::RoaringBitmap; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use super::VectorStoreStats; use crate::score_details::{self, ScoreDetails}; -use crate::vector::{DistributionShift, Embedder, HannoyWrapper}; +use crate::vector::{DistributionShift, Embedder, VectorStore}; use crate::{DocumentId, Result, SearchContext, SearchLogger}; pub struct VectorSort { @@ -56,8 +56,7 @@ impl VectorSort { let target = &self.target; let before = Instant::now(); - let reader = - HannoyWrapper::new(ctx.index.vector_hannoy, self.embedder_index, self.quantized); + let reader = VectorStore::new(ctx.index.vector_hannoy, self.embedder_index, self.quantized); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 2e70958c0..ec3a5a565 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use roaring::RoaringBitmap; use crate::score_details::{self, ScoreDetails}; -use crate::vector::{Embedder, HannoyWrapper}; +use crate::vector::{Embedder, VectorStore}; use crate::{filtered_universe, DocumentId, Filter, Index, Result, SearchResult}; pub struct Similar<'a> { @@ -72,7 +72,7 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let reader = HannoyWrapper::new(self.index.vector_hannoy, embedder_index, self.quantized); + let reader = VectorStore::new(self.index.vector_hannoy, embedder_index, self.quantized); let results = reader.nns_by_item( self.rtxn, self.id, diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index ec1aac32a..5bfc8c218 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -39,7 +39,7 @@ use crate::update::{ IndexerConfig, UpdateIndexingStep, WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst, }; use crate::vector::db::EmbedderInfo; -use crate::vector::{HannoyWrapper, RuntimeEmbedders}; +use crate::vector::{RuntimeEmbedders, VectorStore}; use crate::{CboRoaringBitmapCodec, Index, Result, UserError}; static MERGED_DATABASE_COUNT: usize = 7; @@ -494,7 +494,7 @@ where }, )?; let reader = - HannoyWrapper::new(self.index.vector_hannoy, index, action.was_quantized); + VectorStore::new(self.index.vector_hannoy, index, action.was_quantized); let Some(dim) = reader.dimensions(self.wtxn)? else { continue; }; @@ -523,7 +523,7 @@ where let is_quantizing = embedder_config.is_some_and(|action| action.is_being_quantized); pool.install(|| { - let mut writer = HannoyWrapper::new(vector_hannoy, embedder_index, was_quantized); + let mut writer = VectorStore::new(vector_hannoy, embedder_index, was_quantized); writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index fcb5b00d1..b7c936a82 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -32,7 +32,7 @@ use crate::update::settings::{InnerIndexSettings, InnerIndexSettingsDiff}; use crate::update::{AvailableIds, UpdateIndexingStep}; use crate::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use crate::vector::settings::{RemoveFragments, WriteBackToDocuments}; -use crate::vector::HannoyWrapper; +use crate::vector::VectorStore; use crate::{FieldDistribution, FieldId, FieldIdMapMissingEntry, Index, Result}; pub struct TransformOutput { @@ -834,14 +834,14 @@ impl<'a, 'i> Transform<'a, 'i> { None }; - let readers: BTreeMap<&str, (HannoyWrapper, &RoaringBitmap)> = settings_diff + let readers: BTreeMap<&str, (VectorStore, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() .filter_map(|(name, action)| { if let Some(WriteBackToDocuments { embedder_id, user_provided }) = action.write_back() { - let reader = HannoyWrapper::new( + let reader = VectorStore::new( self.index.vector_hannoy, *embedder_id, action.was_quantized, @@ -950,7 +950,7 @@ impl<'a, 'i> Transform<'a, 'i> { continue; }; let hannoy = - HannoyWrapper::new(self.index.vector_hannoy, infos.embedder_id, was_quantized); + VectorStore::new(self.index.vector_hannoy, infos.embedder_id, was_quantized); let Some(dimensions) = hannoy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index b0590eab4..31616906c 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -27,7 +27,7 @@ use crate::update::index_documents::helpers::{ }; use crate::update::settings::InnerIndexSettingsDiff; use crate::vector::db::{EmbeddingStatusDelta, IndexEmbeddingConfig}; -use crate::vector::HannoyWrapper; +use crate::vector::VectorStore; use crate::{ lat_lng_to_xyz, CboRoaringBitmapCodec, DocumentId, FieldId, GeoPoint, Index, InternalError, Result, SerializationError, U8StrStrCodec, @@ -677,8 +677,7 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .is_some_and(|conf| conf.is_quantized); // FIXME: allow customizing distance - let writer = - HannoyWrapper::new(index.vector_hannoy, infos.embedder_id, binary_quantized); + let writer = VectorStore::new(index.vector_hannoy, infos.embedder_id, binary_quantized); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 35db7b9fc..02d7e10b7 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -24,7 +24,7 @@ use crate::progress::{EmbedderStats, Progress}; use crate::update::settings::SettingsDelta; use crate::update::GrenadParameters; use crate::vector::settings::{EmbedderAction, RemoveFragments, WriteBackToDocuments}; -use crate::vector::{Embedder, HannoyWrapper, RuntimeEmbedders}; +use crate::vector::{Embedder, RuntimeEmbedders, VectorStore}; use crate::{FieldsIdsMap, GlobalFieldsIdsMap, Index, InternalError, Result, ThreadPoolNoAbort}; pub(crate) mod de; @@ -144,7 +144,7 @@ where })?; let dimensions = runtime.embedder.dimensions(); - let writer = HannoyWrapper::new(vector_arroy, embedder_index, runtime.is_quantized); + let writer = VectorStore::new(vector_arroy, embedder_index, runtime.is_quantized); Ok(( embedder_index, @@ -342,7 +342,7 @@ fn hannoy_writers_from_embedder_actions<'indexer>( embedder_actions: &'indexer BTreeMap, embedders: &'indexer RuntimeEmbedders, index_embedder_category_ids: &'indexer std::collections::HashMap, -) -> Result> { +) -> Result> { let vector_arroy = index.vector_hannoy; embedders @@ -362,7 +362,7 @@ fn hannoy_writers_from_embedder_actions<'indexer>( ))); }; let writer = - HannoyWrapper::new(vector_arroy, embedder_category_id, action.was_quantized); + VectorStore::new(vector_arroy, embedder_category_id, action.was_quantized); let dimensions = runtime.embedder.dimensions(); Some(Ok(( embedder_category_id, @@ -385,7 +385,7 @@ where let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; }; - let reader = HannoyWrapper::new(index.vector_hannoy, *embedder_id, action.was_quantized); + let reader = VectorStore::new(index.vector_hannoy, *embedder_id, action.was_quantized); let Some(dimensions) = reader.dimensions(wtxn)? else { continue; }; @@ -401,7 +401,7 @@ where let Some(infos) = index.embedding_configs().embedder_info(wtxn, embedder_name)? else { continue; }; - let arroy = HannoyWrapper::new(index.vector_hannoy, infos.embedder_id, was_quantized); + let arroy = VectorStore::new(index.vector_hannoy, infos.embedder_id, was_quantized); let Some(dimensions) = arroy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/new/indexer/write.rs b/crates/milli/src/update/new/indexer/write.rs index 4be916c02..a023e1431 100644 --- a/crates/milli/src/update/new/indexer/write.rs +++ b/crates/milli/src/update/new/indexer/write.rs @@ -15,7 +15,7 @@ use crate::progress::Progress; use crate::update::settings::InnerIndexSettings; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::settings::EmbedderAction; -use crate::vector::{Embedder, Embeddings, HannoyWrapper, RuntimeEmbedders}; +use crate::vector::{Embedder, Embeddings, RuntimeEmbedders, VectorStore}; use crate::{Error, Index, InternalError, Result, UserError}; pub fn write_to_db( @@ -23,7 +23,7 @@ pub fn write_to_db( finished_extraction: &AtomicBool, index: &Index, wtxn: &mut RwTxn<'_>, - hannoy_writers: &HashMap, + hannoy_writers: &HashMap, ) -> Result { // Used by by the HannoySetVector to copy the embedding into an // aligned memory area, required by arroy to accept a new vector. @@ -116,7 +116,7 @@ pub fn build_vectors( progress: &Progress, index_embeddings: Vec, hannoy_memory: Option, - hannoy_writers: &mut HashMap, + hannoy_writers: &mut HashMap, embeder_actions: Option<&BTreeMap>, must_stop_processing: &MSP, ) -> Result<()> @@ -181,7 +181,7 @@ pub fn write_from_bbqueue( writer_receiver: &mut WriterBbqueueReceiver<'_>, index: &Index, wtxn: &mut RwTxn<'_>, - hannoy_writers: &HashMap, + hannoy_writers: &HashMap, aligned_embedding: &mut Vec, ) -> crate::Result<()> { while let Some(frame_with_header) = writer_receiver.recv_frame() { diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index 64e1377ad..a091f5ab9 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -14,7 +14,7 @@ use crate::constants::RESERVED_VECTORS_FIELD_NAME; use crate::documents::FieldIdMapper; use crate::vector::db::{EmbeddingStatus, IndexEmbeddingConfig}; use crate::vector::parsed_vectors::{RawVectors, RawVectorsError, VectorOrArrayOfVectors}; -use crate::vector::{Embedding, HannoyWrapper, RuntimeEmbedders}; +use crate::vector::{Embedding, RuntimeEmbedders, VectorStore}; use crate::{DocumentId, Index, InternalError, Result, UserError}; #[derive(Serialize)] @@ -121,7 +121,7 @@ impl<'t> VectorDocumentFromDb<'t> { status: &EmbeddingStatus, ) -> Result> { let reader = - HannoyWrapper::new(self.index.vector_hannoy, embedder_id, config.config.quantized()); + VectorStore::new(self.index.vector_hannoy, embedder_id, config.config.quantized()); let vectors = reader.item_vectors(self.rtxn, self.docid)?; Ok(VectorEntry { diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 512361029..649412ade 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -45,13 +45,13 @@ const HANNOY_EF_CONSTRUCTION: usize = 48; const HANNOY_M: usize = 16; const HANNOY_M0: usize = 32; -pub struct HannoyWrapper { +pub struct VectorStore { quantized: bool, embedder_index: u8, database: hannoy::Database, } -impl HannoyWrapper { +impl VectorStore { pub fn new( database: hannoy::Database, embedder_index: u8, From a7cd6853dbef61d8f6130ada798731cefbcfa3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 29 Jul 2025 17:31:33 +0200 Subject: [PATCH 04/62] Rename the vector store const name and keep the vector-arroy db name --- crates/meilitool/src/upgrade/v1_11.rs | 4 ++-- crates/milli/src/index.rs | 2 +- crates/milli/src/update/new/vector_document.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index 385487b89..e8327a26d 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -68,7 +68,7 @@ pub fn v1_10_to_v1_11( ) })?; let index_read_database = - try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_HANNOY) + try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_STORE) .with_context(|| format!("while updating date format for index `{uid}`"))?; let mut index_wtxn = index_env.write_txn().with_context(|| { @@ -79,7 +79,7 @@ pub fn v1_10_to_v1_11( })?; let index_write_database = - try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_HANNOY) + try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_STORE) .with_context(|| format!("while updating date format for index `{uid}`"))?; // meilisearch_types::milli::hannoy::upgrade::cosine_from_0_4_to_0_5( diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index c4651b5a9..d76ecfc9d 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -113,7 +113,7 @@ pub mod db_name { pub const FIELD_ID_DOCID_FACET_F64S: &str = "field-id-docid-facet-f64s"; pub const FIELD_ID_DOCID_FACET_STRINGS: &str = "field-id-docid-facet-strings"; pub const VECTOR_EMBEDDER_CATEGORY_ID: &str = "vector-embedder-category-id"; - pub const VECTOR_HANNOY: &str = "vector-hannoy"; + pub const VECTOR_STORE: &str = "vector-arroy"; pub const DOCUMENTS: &str = "documents"; } const NUMBER_OF_DBS: u32 = 25; diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index a091f5ab9..af171f143 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -149,7 +149,7 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> { name, entry_from_raw_value(value, false).map_err(|_| { InternalError::Serialization(crate::SerializationError::Decoding { - db_name: Some(crate::index::db_name::VECTOR_HANNOY), + db_name: Some(crate::index::db_name::VECTOR_STORE), }) })?, )) @@ -167,7 +167,7 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> { Some(embedding_from_doc) => { Some(entry_from_raw_value(embedding_from_doc, false).map_err(|_| { InternalError::Serialization(crate::SerializationError::Decoding { - db_name: Some(crate::index::db_name::VECTOR_HANNOY), + db_name: Some(crate::index::db_name::VECTOR_STORE), }) })?) } From 27550dafad4517345d47e48ebe7361580151ae66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 29 Jul 2025 18:00:29 +0200 Subject: [PATCH 05/62] Reintroduce arroy and support for dumpless upgrade from previous versions --- Cargo.lock | 32 +++++++++++++++++++ crates/meilitool/src/main.rs | 4 +-- crates/meilitool/src/upgrade/v1_11.rs | 13 ++++---- crates/milli/Cargo.toml | 1 + crates/milli/src/error.rs | 24 ++++++++++++++ crates/milli/src/index.rs | 10 +++--- crates/milli/src/lib.rs | 2 +- crates/milli/src/search/new/vector_sort.rs | 2 +- crates/milli/src/search/similar.rs | 2 +- crates/milli/src/update/clear_documents.rs | 7 ++-- .../milli/src/update/index_documents/mod.rs | 5 ++- .../src/update/index_documents/transform.rs | 4 +-- .../src/update/index_documents/typed_chunk.rs | 2 +- crates/milli/src/update/new/indexer/mod.rs | 8 ++--- .../milli/src/update/new/vector_document.rs | 2 +- crates/milli/src/update/upgrade/v1_14.rs | 15 ++++----- 16 files changed, 94 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 378d08124..fdacb4471 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -442,6 +442,28 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arroy" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e6111f351d004bd13e95ab540721272136fd3218b39d3ec95a2ea1c4e6a0a6" +dependencies = [ + "bytemuck", + "byteorder", + "enum-iterator", + "heed", + "memmap2", + "nohash", + "ordered-float 4.6.0", + "page_size", + "rand 0.8.5", + "rayon", + "roaring", + "tempfile", + "thiserror 2.0.12", + "tracing", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -3927,6 +3949,7 @@ name = "milli" version = "1.19.0" dependencies = [ "allocator-api2 0.3.0", + "arroy", "bbqueue", "big_s", "bimap", @@ -4382,6 +4405,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.0.0" diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 439bba015..1a2110b5d 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -660,10 +660,10 @@ fn hair_dryer( match part { IndexPart::Hannoy => { let mut count = 0; - let total = index.vector_hannoy.len(&rtxn)?; + let total = index.vector_store.len(&rtxn)?; eprintln!("Hair drying hannoy for {uid}..."); for (i, result) in index - .vector_hannoy + .vector_store .remap_types::() .iter(&rtxn)? .enumerate() diff --git a/crates/meilitool/src/upgrade/v1_11.rs b/crates/meilitool/src/upgrade/v1_11.rs index e8327a26d..f1d5c1959 100644 --- a/crates/meilitool/src/upgrade/v1_11.rs +++ b/crates/meilitool/src/upgrade/v1_11.rs @@ -82,13 +82,12 @@ pub fn v1_10_to_v1_11( try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_STORE) .with_context(|| format!("while updating date format for index `{uid}`"))?; - // meilisearch_types::milli::hannoy::upgrade::cosine_from_0_4_to_0_5( - // &index_rtxn, - // index_read_database.remap_types(), - // &mut index_wtxn, - // index_write_database.remap_types(), - // )?; - unimplemented!("Hannoy doesn't support upgrading"); + meilisearch_types::milli::arroy::upgrade::cosine_from_0_4_to_0_5( + &index_rtxn, + index_read_database.remap_types(), + &mut index_wtxn, + index_write_database.remap_types(), + )?; index_wtxn.commit()?; } diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 812ac4376..b24a1779f 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -87,6 +87,7 @@ rhai = { version = "1.22.2", features = [ "no_time", "sync", ] } +arroy = "0.6.1" hannoy = { git = "https://github.com/nnethercott/hannoy", tag = "v0.0.1" } rand = "0.8.5" tracing = "0.1.41" diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 2769284aa..787f42753 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -76,6 +76,8 @@ pub enum InternalError { #[error("Cannot upgrade to the following version: v{0}.{1}.{2}.")] CannotUpgradeToVersion(u32, u32, u32), #[error(transparent)] + ArroyError(#[from] arroy::Error), + #[error(transparent)] HannoyError(#[from] hannoy::Error), #[error(transparent)] VectorEmbeddingError(#[from] crate::vector::Error), @@ -419,6 +421,28 @@ impl From for Error { } } +impl From for Error { + fn from(value: arroy::Error) -> Self { + match value { + arroy::Error::Heed(heed) => heed.into(), + arroy::Error::Io(io) => io.into(), + arroy::Error::InvalidVecDimension { expected, received } => { + Error::UserError(UserError::InvalidVectorDimensions { expected, found: received }) + } + arroy::Error::BuildCancelled => Error::InternalError(InternalError::AbortedIndexation), + arroy::Error::DatabaseFull + | arroy::Error::InvalidItemAppend + | arroy::Error::UnmatchingDistance { .. } + | arroy::Error::NeedBuild(_) + | arroy::Error::MissingKey { .. } + | arroy::Error::MissingMetadata(_) + | arroy::Error::CannotDecodeKeyMode { .. } => { + Error::InternalError(InternalError::ArroyError(value)) + } + } + } +} + impl From for Error { fn from(value: hannoy::Error) -> Self { match value { diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index d76ecfc9d..dc8298a1b 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -180,7 +180,7 @@ pub struct Index { /// Maps an embedder name to its id in the hannoy store. pub(crate) embedder_category_id: Database, /// Vector store based on hannoy™. - pub vector_hannoy: hannoy::Database, + pub vector_store: hannoy::Database, /// Maps the document id to the document as an obkv store. pub(crate) documents: Database, @@ -264,7 +264,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_hannoy, + vector_store: vector_hannoy, embedder_category_id, documents, }; @@ -1773,7 +1773,7 @@ impl Index { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); let has_fragments = config.config.embedder_options.has_fragments(); let reader = VectorStore::new( - self.vector_hannoy, + self.vector_store, embedder_info.embedder_id, config.config.quantized(), ); @@ -1798,7 +1798,7 @@ impl Index { for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); let reader = - VectorStore::new(self.vector_hannoy, embedder_id, config.config.quantized()); + VectorStore::new(self.vector_store, embedder_id, config.config.quantized()); reader.aggregate_stats(rtxn, &mut stats)?; } Ok(stats) @@ -1842,7 +1842,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_hannoy, + vector_store: vector_hannoy, embedder_category_id, documents, } = self; diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index 91ea87b68..ca867d6e0 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -53,7 +53,7 @@ pub use search::new::{ }; use serde_json::Value; pub use thread_pool_no_abort::{PanicCatched, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder}; -pub use {charabia as tokenizer, hannoy, heed, rhai}; +pub use {arroy, charabia as tokenizer, hannoy, heed, rhai}; pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError}; pub use self::attribute_patterns::{AttributePatterns, PatternMatch}; diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index ce755c57d..284fcd431 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -56,7 +56,7 @@ impl VectorSort { let target = &self.target; let before = Instant::now(); - let reader = VectorStore::new(ctx.index.vector_hannoy, self.embedder_index, self.quantized); + let reader = VectorStore::new(ctx.index.vector_store, self.embedder_index, self.quantized); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index ec3a5a565..83e65fd6a 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -72,7 +72,7 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let reader = VectorStore::new(self.index.vector_hannoy, embedder_index, self.quantized); + let reader = VectorStore::new(self.index.vector_store, embedder_index, self.quantized); let results = reader.nns_by_item( self.rtxn, self.id, diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index 164aed9a0..6cd389d42 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -2,7 +2,8 @@ use heed::RwTxn; use roaring::RoaringBitmap; use time::OffsetDateTime; -use crate::{database_stats::DatabaseStats, FieldDistribution, Index, Result}; +use crate::database_stats::DatabaseStats; +use crate::{FieldDistribution, Index, Result}; pub struct ClearDocuments<'t, 'i> { wtxn: &'t mut RwTxn<'i>, @@ -45,7 +46,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_hannoy, + vector_store, embedder_category_id: _, documents, } = self.index; @@ -88,7 +89,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { field_id_docid_facet_f64s.clear(self.wtxn)?; field_id_docid_facet_strings.clear(self.wtxn)?; // vector - vector_hannoy.clear(self.wtxn)?; + vector_store.clear(self.wtxn)?; documents.clear(self.wtxn)?; diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 5bfc8c218..f4d1552bd 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -493,8 +493,7 @@ where key: None, }, )?; - let reader = - VectorStore::new(self.index.vector_hannoy, index, action.was_quantized); + let reader = VectorStore::new(self.index.vector_store, index, action.was_quantized); let Some(dim) = reader.dimensions(self.wtxn)? else { continue; }; @@ -504,7 +503,7 @@ where for (embedder_name, dimension) in dimension { let wtxn = &mut *self.wtxn; - let vector_hannoy = self.index.vector_hannoy; + let vector_hannoy = self.index.vector_store; let cancel = &self.should_abort; let embedder_index = diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index b7c936a82..985f3a88f 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -842,7 +842,7 @@ impl<'a, 'i> Transform<'a, 'i> { action.write_back() { let reader = VectorStore::new( - self.index.vector_hannoy, + self.index.vector_store, *embedder_id, action.was_quantized, ); @@ -950,7 +950,7 @@ impl<'a, 'i> Transform<'a, 'i> { continue; }; let hannoy = - VectorStore::new(self.index.vector_hannoy, infos.embedder_id, was_quantized); + VectorStore::new(self.index.vector_store, infos.embedder_id, was_quantized); let Some(dimensions) = hannoy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index 31616906c..fe5a8bde8 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -677,7 +677,7 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .is_some_and(|conf| conf.is_quantized); // FIXME: allow customizing distance - let writer = VectorStore::new(index.vector_hannoy, infos.embedder_id, binary_quantized); + let writer = VectorStore::new(index.vector_store, infos.embedder_id, binary_quantized); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 02d7e10b7..b115761cd 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -130,7 +130,7 @@ where let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map); - let vector_arroy = index.vector_hannoy; + let vector_arroy = index.vector_store; let hannoy_writers: Result> = embedders .inner_as_ref() .iter() @@ -343,7 +343,7 @@ fn hannoy_writers_from_embedder_actions<'indexer>( embedders: &'indexer RuntimeEmbedders, index_embedder_category_ids: &'indexer std::collections::HashMap, ) -> Result> { - let vector_arroy = index.vector_hannoy; + let vector_arroy = index.vector_store; embedders .inner_as_ref() @@ -385,7 +385,7 @@ where let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; }; - let reader = VectorStore::new(index.vector_hannoy, *embedder_id, action.was_quantized); + let reader = VectorStore::new(index.vector_store, *embedder_id, action.was_quantized); let Some(dimensions) = reader.dimensions(wtxn)? else { continue; }; @@ -401,7 +401,7 @@ where let Some(infos) = index.embedding_configs().embedder_info(wtxn, embedder_name)? else { continue; }; - let arroy = VectorStore::new(index.vector_hannoy, infos.embedder_id, was_quantized); + let arroy = VectorStore::new(index.vector_store, infos.embedder_id, was_quantized); let Some(dimensions) = arroy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index af171f143..d04f9bb79 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -121,7 +121,7 @@ impl<'t> VectorDocumentFromDb<'t> { status: &EmbeddingStatus, ) -> Result> { let reader = - VectorStore::new(self.index.vector_hannoy, embedder_id, config.config.quantized()); + VectorStore::new(self.index.vector_store, embedder_id, config.config.quantized()); let vectors = reader.item_vectors(self.rtxn, self.docid)?; Ok(VectorEntry { diff --git a/crates/milli/src/update/upgrade/v1_14.rs b/crates/milli/src/update/upgrade/v1_14.rs index 68ff9f8cd..9950be706 100644 --- a/crates/milli/src/update/upgrade/v1_14.rs +++ b/crates/milli/src/update/upgrade/v1_14.rs @@ -1,4 +1,4 @@ -use hannoy::distances::Cosine; +use arroy::distances::Cosine; use heed::RwTxn; use super::UpgradeIndex; @@ -25,13 +25,12 @@ impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 { progress.update_progress(VectorStore::UpdateInternalVersions); let rtxn = index.read_txn()?; - // hannoy::upgrade::from_0_5_to_0_6::( - // &rtxn, - // index.vector_hannoy.remap_data_type(), - // wtxn, - // index.vector_hannoy.remap_data_type(), - // )?; - unimplemented!("upgrade hannoy"); + arroy::upgrade::from_0_5_to_0_6::( + &rtxn, + index.vector_store.remap_types(), + wtxn, + index.vector_store.remap_types(), + )?; Ok(false) } From e50f970ab8615147522e5e181138d93a0509c257 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 30 Jul 2025 09:39:55 +0200 Subject: [PATCH 06/62] Use a more feature-full Hannoy version --- Cargo.lock | 7 +------ crates/milli/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdacb4471..64cfe2cd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,23 +2604,18 @@ dependencies = [ [[package]] name = "hannoy" version = "0.0.1" -source = "git+https://github.com/nnethercott/hannoy?tag=v0.0.1#d51750cd5612b6875375f5f4ad3928c87d55ee38" +source = "git+https://github.com/Kerollmops/hannoy?branch=release-0.0.2#f016685623dadf20bd75df0748b45643d326b146" dependencies = [ "bytemuck", "byteorder", - "enum-iterator", "hashbrown 0.15.4", "heed", - "memmap2", "min-max-heap", - "nohash", - "ordered-float 5.0.0", "page_size", "papaya", "rand 0.8.5", "rayon", "roaring", - "tempfile", "thiserror 2.0.12", "tinyvec", "tracing", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index b24a1779f..3ff7cb323 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.1" -hannoy = { git = "https://github.com/nnethercott/hannoy", tag = "v0.0.1" } +hannoy = { git = "https://github.com/Kerollmops/hannoy", branch = "release-0.0.2" } rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } From f9d0d1ddd640b151d8b2bc7312291dc032b58eea Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 30 Jul 2025 17:24:55 +0200 Subject: [PATCH 07/62] Bump hannoy --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64cfe2cd4..66ee9c3cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "hannoy" version = "0.0.1" -source = "git+https://github.com/Kerollmops/hannoy?branch=release-0.0.2#f016685623dadf20bd75df0748b45643d326b146" +source = "git+https://github.com/Kerollmops/hannoy?branch=release-0.0.2#d734d5ab13ae34b3ee4d34b9eeb3ed8843cd11c2" dependencies = [ "bytemuck", "byteorder", @@ -3361,7 +3361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] From 6176b143bbe55c2ee99f2c90757e6a01767a8534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 31 Jul 2025 11:40:15 +0200 Subject: [PATCH 08/62] remove-me: Introduce an env var to change the embeddings chunk size --- crates/milli/src/vector/rest.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/vector/rest.rs b/crates/milli/src/vector/rest.rs index 7a16f1a1e..64e4e74c7 100644 --- a/crates/milli/src/vector/rest.rs +++ b/crates/milli/src/vector/rest.rs @@ -321,7 +321,14 @@ impl Embedder { pub fn prompt_count_in_chunk_hint(&self) -> usize { match self.data.request.input_type() { InputType::Text => 1, - InputType::TextArray => 10, + InputType::TextArray => { + let chunk_size = std::env::var("MEILI_EMBEDDINGS_CHUNK_SIZE") + .ok() + .and_then(|chunk_size| chunk_size.parse().ok()) + .unwrap_or(10); + assert!(chunk_size <= 100, "Embedding chunk size cannot exceed 100"); + chunk_size + } } } From 2b2559016ae7353e89c2544f31be7e8013ff6707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 4 Aug 2025 11:47:38 +0200 Subject: [PATCH 09/62] Increase efConstruction from 48 to 125 --- crates/milli/src/vector/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 649412ade..a038b7c6c 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -41,7 +41,7 @@ pub type Embedding = Vec; pub const REQUEST_PARALLELISM: usize = 40; pub const MAX_COMPOSITE_DISTANCE: f32 = 0.01; -const HANNOY_EF_CONSTRUCTION: usize = 48; +const HANNOY_EF_CONSTRUCTION: usize = 125; const HANNOY_M: usize = 16; const HANNOY_M0: usize = 32; From 493d67ffd4a458b867fface0a9c7f16de7500f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 4 Aug 2025 11:47:56 +0200 Subject: [PATCH 10/62] Increase efSearch from x2 to x10 --- crates/milli/src/vector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index a038b7c6c..6c016eccd 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -468,7 +468,7 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit, limit * 2); // TODO find better ef + let mut searcher = reader.nns(limit, limit * 10); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; @@ -510,7 +510,7 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit, limit * 2); // TODO find better ef + let mut searcher = reader.nns(limit, limit * 10); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; From 47cee7e1eafb9aeb94eedf6d65ec0430a0b482dd Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Aug 2025 15:08:14 +0200 Subject: [PATCH 11/62] Bump Hannoy's version --- Cargo.lock | 4 ++-- crates/milli/Cargo.toml | 2 +- crates/milli/src/vector/mod.rs | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66ee9c3cb..3f69af792 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2603,8 +2603,8 @@ dependencies = [ [[package]] name = "hannoy" -version = "0.0.1" -source = "git+https://github.com/Kerollmops/hannoy?branch=release-0.0.2#d734d5ab13ae34b3ee4d34b9eeb3ed8843cd11c2" +version = "0.0.2" +source = "git+https://github.com/nnethercott/hannoy?rev=cab22c0#cab22c01dce4c825ae4d0d8824ba51ed4cef57e1" dependencies = [ "bytemuck", "byteorder", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 3ff7cb323..12dacf1b1 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.1" -hannoy = { git = "https://github.com/Kerollmops/hannoy", branch = "release-0.0.2" } +hannoy = { git = "https://github.com/nnethercott/hannoy", rev = "cab22c0" } rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 6c016eccd..5ff4f658e 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -468,7 +468,8 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit, limit * 10); // TODO find better ef + let mut searcher = reader.nns(limit); + searcher.ef_search(limit * 10); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; @@ -510,7 +511,8 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; - let mut searcher = reader.nns(limit, limit * 10); // TODO find better ef + let mut searcher = reader.nns(limit); + searcher.ef_search(limit * 10); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; From 30110a04885291072b075c807f92e3b77c21da72 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Aug 2025 15:08:48 +0200 Subject: [PATCH 12/62] Reintroduce changing the distance from Cosine to Cosine binary quantized --- crates/milli/src/vector/mod.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 5ff4f658e..3a70d717f 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -166,20 +166,20 @@ impl VectorStore { // only happens once in the life of an embedder, it's not very performances // sensitive. if quantizing && !self.quantized { - // let writer = writer.prepare_changing_distance::(wtxn)?; - // writer - // .builder(rng) - // .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - // .progress(|step| progress.update_progress_from_hannoy(step)) - // .cancel(cancel) - // .build(wtxn)?; - unimplemented!("switching from quantized to non-quantized"); + let writer = writer.prepare_changing_distance::(wtxn)?; + writer + .builder(rng) + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + // .progress(|step| progress.update_progress_from_hannoy(step)) + .cancel(cancel) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; } else if writer.need_build(wtxn)? { writer .builder(rng) .available_memory(hannoy_memory.unwrap_or(usize::MAX)) // .progress(|step| progress.update_progress_from_hannoy(step)) - // .cancel(cancel) + .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; } else if writer.is_empty(wtxn)? { From 6d92c94bb32ed9a01f6295f984f545f5676bd2de Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 6 Aug 2025 15:11:59 +0200 Subject: [PATCH 13/62] Add a missing cancelation call for hannoy --- crates/milli/src/vector/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 3a70d717f..854ce9e74 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -153,6 +153,8 @@ impl VectorStore { if writer.need_build(wtxn)? { writer .builder(rng) + // .progress(|step| progress.update_progress_from_hannoy(step)) + .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)? } else if writer.is_empty(wtxn)? { From 52d55ccd8e9fcf8873f21210f160fc8b279bf7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 7 Aug 2025 12:03:24 +0200 Subject: [PATCH 14/62] Switch to hannoy with support for deletions --- Cargo.lock | 3 ++- crates/milli/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f69af792..f38557003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "hannoy" version = "0.0.2" -source = "git+https://github.com/nnethercott/hannoy?rev=cab22c0#cab22c01dce4c825ae4d0d8824ba51ed4cef57e1" +source = "git+https://github.com/nnethercott/hannoy?branch=main#93a24c4cdf712152c90d27a2898715f22942c35c" dependencies = [ "bytemuck", "byteorder", @@ -2616,6 +2616,7 @@ dependencies = [ "rand 0.8.5", "rayon", "roaring", + "rustc-hash 2.1.1", "thiserror 2.0.12", "tinyvec", "tracing", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 12dacf1b1..1ae431082 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.1" -hannoy = { git = "https://github.com/nnethercott/hannoy", rev = "cab22c0" } +hannoy = { git = "https://github.com/nnethercott/hannoy", branch = "main" } rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } From 0b3f983d27abd3a744b8623ac36f7f0f891551f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 7 Aug 2025 14:34:57 +0200 Subject: [PATCH 15/62] Always use at least an ef = 100 when searching --- crates/milli/src/vector/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 854ce9e74..50c947f12 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -471,7 +471,7 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); - searcher.ef_search(limit * 10); // TODO find better ef + searcher.ef_search((limit * 10).max(100)); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; @@ -514,7 +514,7 @@ impl VectorStore { for reader in self.readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); - searcher.ef_search(limit * 10); // TODO find better ef + searcher.ef_search((limit * 10).max(100)); // TODO find better ef if let Some(filter) = filter { if reader.item_ids().is_disjoint(filter) { continue; From aef07f4bfaa03d1be04c8ad1652ba803260a5fe3 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Fri, 8 Aug 2025 09:47:08 +0200 Subject: [PATCH 16/62] wip: Use Hamming when binary quantized --- crates/milli/src/vector/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 50c947f12..ddc5ebffa 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use deserr::{DeserializeError, Deserr}; -use hannoy::distances::{BinaryQuantizedCosine, Cosine}; +use hannoy::distances::{Cosine, Hamming}; use hannoy::ItemId; use heed::{RoTxn, RwTxn, Unspecified}; use ordered_float::OrderedFloat; @@ -168,7 +168,7 @@ impl VectorStore { // only happens once in the life of an embedder, it's not very performances // sensitive. if quantizing && !self.quantized { - let writer = writer.prepare_changing_distance::(wtxn)?; + let writer = writer.prepare_changing_distance::(wtxn)?; writer .builder(rng) .available_memory(hannoy_memory.unwrap_or(usize::MAX)) @@ -553,7 +553,7 @@ impl VectorStore { self.database.remap_data_type() } - fn quantized_db(&self) -> hannoy::Database { + fn quantized_db(&self) -> hannoy::Database { self.database.remap_data_type() } From 0d4b78a217adc42d1d311265ceb2c47020048eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 11 Aug 2025 18:05:17 +0200 Subject: [PATCH 17/62] Integrate the hannoy progress --- Cargo.lock | 18 +++++++++-- crates/milli/Cargo.toml | 1 + crates/milli/src/progress.rs | 32 ++++++++++++++----- .../milli/src/update/index_documents/mod.rs | 2 +- crates/milli/src/update/new/indexer/write.rs | 2 +- crates/milli/src/vector/mod.rs | 20 ++++++------ 6 files changed, 51 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f38557003..54531907a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "hannoy" version = "0.0.2" -source = "git+https://github.com/nnethercott/hannoy?branch=main#93a24c4cdf712152c90d27a2898715f22942c35c" +source = "git+https://github.com/nnethercott/hannoy?branch=main#8d1846b188ed2cc8776fdb86805eefbfbde9ddd1" dependencies = [ "bytemuck", "byteorder", @@ -2617,6 +2617,7 @@ dependencies = [ "rayon", "roaring", "rustc-hash 2.1.1", + "steppe", "thiserror 2.0.12", "tinyvec", "tracing", @@ -3056,9 +3057,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown 0.15.4", @@ -4005,6 +4006,7 @@ dependencies = [ "smallstr", "smallvec", "smartstring", + "steppe", "tempfile", "thiserror 2.0.12", "thread_local", @@ -5866,6 +5868,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "steppe" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dead99cdf718f37bcd1d22dda9b498f35c5aa22894b755bfd94bf8c2daec9427" +dependencies = [ + "convert_case 0.8.0", + "indexmap", +] + [[package]] name = "strsim" version = "0.10.0" diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 1ae431082..009e874fa 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -96,6 +96,7 @@ url = "2.5.4" hashbrown = "0.15.4" bumpalo = "3.18.1" bumparaw-collections = "0.1.4" +steppe = { version = "0.4.0", default-features = false } thread_local = "1.1.9" allocator-api2 = "0.3.0" rustc-hash = "2.1.1" diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index 2ec4fd6de..fd56c46d6 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -96,14 +96,6 @@ impl Progress { durations.drain(..).map(|(name, duration)| (name, format!("{duration:.2?}"))).collect() } - - // TODO: ideally we should expose the progress in a way that let arroy use it directly - // pub(crate) fn update_progress_from_hannoy(&self, progress: hannoy::WriterProgress) { - // self.update_progress(progress.main); - // if let Some(sub) = progress.sub { - // self.update_progress(sub); - // } - // } } /// Generate the names associated with the durations and push them. @@ -317,3 +309,27 @@ impl Step for VariableNameStep { // self.max // } // } + +// Integration with steppe + +impl steppe::Progress for Progress { + fn update(&self, sub_progress: impl steppe::Step) { + self.update_progress(Compat(sub_progress)); + } +} + +struct Compat(T); + +impl Step for Compat { + fn name(&self) -> Cow<'static, str> { + self.0.name().into() + } + + fn current(&self) -> u32 { + self.0.current().try_into().unwrap_or(u32::MAX) + } + + fn total(&self) -> u32 { + self.0.total().try_into().unwrap_or(u32::MAX) + } +} diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index f4d1552bd..a52cc413c 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -526,7 +526,7 @@ where writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share - &Progress::default(), + Progress::default(), &mut rng, dimension, is_quantizing, diff --git a/crates/milli/src/update/new/indexer/write.rs b/crates/milli/src/update/new/indexer/write.rs index a023e1431..9cb014e25 100644 --- a/crates/milli/src/update/new/indexer/write.rs +++ b/crates/milli/src/update/new/indexer/write.rs @@ -136,7 +136,7 @@ where .unwrap_or(false); writer.build_and_quantize( wtxn, - progress, + progress.clone(), &mut rng, dimensions, is_being_quantized, diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index ddc5ebffa..bba617782 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -140,7 +140,7 @@ impl VectorStore { pub fn build_and_quantize( &mut self, wtxn: &mut RwTxn, - progress: &Progress, + progress: Progress, rng: &mut R, dimension: usize, quantizing: bool, @@ -151,12 +151,12 @@ impl VectorStore { if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.need_build(wtxn)? { - writer - .builder(rng) - // .progress(|step| progress.update_progress_from_hannoy(step)) + let mut builder = writer.builder(rng).progress(progress.clone()); + builder + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)? + .build::(wtxn)?; } else if writer.is_empty(wtxn)? { continue; } @@ -169,18 +169,16 @@ impl VectorStore { // sensitive. if quantizing && !self.quantized { let writer = writer.prepare_changing_distance::(wtxn)?; - writer - .builder(rng) + let mut builder = writer.builder(rng).progress(progress.clone()); + builder .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - // .progress(|step| progress.update_progress_from_hannoy(step)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; } else if writer.need_build(wtxn)? { - writer - .builder(rng) + let mut builder = writer.builder(rng).progress(progress.clone()); + builder .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - // .progress(|step| progress.update_progress_from_hannoy(step)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; From 6e4dfa016841def8ba5d3d367399ef3ac6166f35 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 18 Aug 2025 16:37:38 +0200 Subject: [PATCH 18/62] First version of Hannoy dumpless upgrade --- crates/milli/src/update/upgrade/mod.rs | 35 ++++++++-- crates/milli/src/update/upgrade/v1_18.rs | 34 ++++++++++ crates/milli/src/vector/mod.rs | 85 +++++++++++++++++++++--- 3 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 crates/milli/src/update/upgrade/v1_18.rs diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 01ad677c7..40826cbe1 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -3,15 +3,18 @@ mod v1_13; mod v1_14; mod v1_15; mod v1_16; +mod v1_18; + use heed::RwTxn; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_15::Latest_V1_14_To_Latest_V1_15; +use v1_16::Latest_V1_15_To_V1_16_0; +use v1_18::Latest_V1_17_To_V1_18_0; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; -use crate::update::upgrade::v1_16::Latest_V1_15_To_V1_16_0; use crate::{Index, InternalError, Result}; trait UpgradeIndex { @@ -34,6 +37,8 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, + &ToTargetNoOp { target: (1, 17, 0) }, + &Latest_V1_17_To_V1_18_0 {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -62,9 +67,9 @@ const fn start(from: (u32, u32, u32)) -> Option { // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), (1, 16, _) => function_index!(7), - (1, 17, _) => function_index!(7), - (1, 18, _) => function_index!(7), - (1, 19, _) => function_index!(7), + (1, 17, _) => function_index!(8), + (1, 18, _) => function_index!(9), + (1, 19, _) => function_index!(9), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, @@ -147,3 +152,25 @@ impl UpgradeIndex for ToCurrentNoOp { (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) } } + +/// Perform no operation during the upgrade except changing to the specified target version. +#[allow(non_camel_case_types)] +struct ToTargetNoOp { + pub target: (u32, u32, u32), +} + +impl UpgradeIndex for ToTargetNoOp { + fn upgrade( + &self, + _wtxn: &mut RwTxn, + _index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + self.target + } +} diff --git a/crates/milli/src/update/upgrade/v1_18.rs b/crates/milli/src/update/upgrade/v1_18.rs new file mode 100644 index 000000000..e20696b84 --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_18.rs @@ -0,0 +1,34 @@ +use heed::RwTxn; + +use super::UpgradeIndex; +use crate::progress::Progress; +use crate::vector::VectorStore; +use crate::{Index, Result}; + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_17_To_V1_18_0(); + +impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + let embedding_configs = index.embedding_configs(); + for config in embedding_configs.embedding_configs(wtxn)? { + // TODO use the embedder name to display progress + let quantized = config.config.quantized(); + let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); + let vector_store = VectorStore::new(index.vector_store, embedder_id, quantized); + vector_store.convert_from_arroy(wtxn)?; + } + + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 18, 0) + } +} diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index bba617782..0a97a9bde 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -8,6 +8,7 @@ use hannoy::distances::{Cosine, Hamming}; use hannoy::ItemId; use heed::{RoTxn, RwTxn, Unspecified}; use ordered_float::OrderedFloat; +use rand::SeedableRng as _; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -64,12 +65,30 @@ impl VectorStore { self.embedder_index } + fn arroy_readers<'a, D: arroy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: arroy::Database, + ) -> impl Iterator, arroy::Error>> + 'a { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match arroy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(arroy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) + } + fn readers<'a, D: hannoy::Distance>( &'a self, rtxn: &'a RoTxn<'a>, db: hannoy::Database, ) -> impl Iterator, hannoy::Error>> + 'a { - hannoy_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { match hannoy::Reader::open(rtxn, index, db) { Ok(reader) => match reader.is_empty(rtxn) { Ok(false) => Some(Ok(reader)), @@ -136,6 +155,46 @@ impl VectorStore { } } + pub fn convert_from_arroy(&self, wtxn: &mut RwTxn) -> crate::Result<()> { + if self.quantized { + let dimensions = self + .arroy_readers(wtxn, self.arroy_quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); + let mut builder = writer.builder(&mut rng); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) + } else { + let dimensions = self + .arroy_readers(wtxn, self.arroy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); + let mut builder = writer.builder(&mut rng); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) + } + } + #[allow(clippy::too_many_arguments)] pub fn build_and_quantize( &mut self, @@ -147,7 +206,7 @@ impl VectorStore { hannoy_memory: Option, cancel: &(impl Fn() -> bool + Sync + Send), ) -> Result<(), hannoy::Error> { - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.need_build(wtxn)? { @@ -202,7 +261,7 @@ impl VectorStore { ) -> Result<(), hannoy::Error> { let dimension = embeddings.dimension(); for (index, vector) in - hannoy_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) + vector_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) { if self.quantized { hannoy::Writer::new(self.quantized_db(), index, dimension) @@ -238,7 +297,7 @@ impl VectorStore { ) -> Result<(), hannoy::Error> { let dimension = vector.len(); - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { let writer = hannoy::Writer::new(db, index, dimension); if !writer.contains_item(wtxn, item_id)? { writer.add_item(wtxn, item_id, vector)?; @@ -287,7 +346,7 @@ impl VectorStore { dimension: usize, item_id: hannoy::ItemId, ) -> Result<(), hannoy::Error> { - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); writer.del_item(wtxn, item_id)?; @@ -387,7 +446,7 @@ impl VectorStore { ) -> Result { let dimension = vector.len(); - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { let writer = hannoy::Writer::new(db, index, dimension); if writer.contains_item(wtxn, item_id)? { return writer.del_item(wtxn, item_id); @@ -397,7 +456,7 @@ impl VectorStore { } pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), hannoy::Error> { - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.is_empty(wtxn)? { @@ -421,7 +480,7 @@ impl VectorStore { dimension: usize, item: hannoy::ItemId, ) -> Result { - for index in hannoy_store_range_for_embedder(self.embedder_index) { + for index in vector_store_range_for_embedder(self.embedder_index) { let contains = if self.quantized { let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); if writer.is_empty(rtxn)? { @@ -547,6 +606,14 @@ impl VectorStore { Ok(vectors) } + fn arroy_angular_db(&self) -> arroy::Database { + self.database.remap_types() + } + + fn arroy_quantized_db(&self) -> arroy::Database { + self.database.remap_types() + } + fn angular_db(&self) -> hannoy::Database { self.database.remap_data_type() } @@ -1230,7 +1297,7 @@ pub const fn is_cuda_enabled() -> bool { cfg!(feature = "cuda") } -fn hannoy_store_range_for_embedder(embedder_id: u8) -> impl Iterator { +fn vector_store_range_for_embedder(embedder_id: u8) -> impl Iterator { (0..=u8::MAX).map(move |store_id| hannoy_store_for_embedder(embedder_id, store_id)) } From 634041221951074a0d8ffd43e45773b6e75972ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 14:24:50 +0200 Subject: [PATCH 19/62] Expose Hannoy progress when upgrading --- crates/milli/src/progress.rs | 43 +----------------------- crates/milli/src/update/upgrade/v1_18.rs | 4 +-- crates/milli/src/vector/mod.rs | 6 ++-- 3 files changed, 6 insertions(+), 47 deletions(-) diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index fd56c46d6..728e95652 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -269,47 +269,6 @@ impl Step for VariableNameStep { } } -// impl Step for hannoy::MainStep { -// fn name(&self) -> Cow<'static, str> { -// match self { -// hannoy::MainStep::PreProcessingTheItems => "pre processing the items", -// hannoy::MainStep::WritingTheDescendantsAndMetadata => { -// "writing the descendants and metadata" -// } -// hannoy::MainStep::RetrieveTheUpdatedItems => "retrieve the updated items", -// hannoy::MainStep::RetrievingTheTreeAndItemNodes => "retrieving the tree and item nodes", -// hannoy::MainStep::UpdatingTheTrees => "updating the trees", -// hannoy::MainStep::CreateNewTrees => "create new trees", -// hannoy::MainStep::WritingNodesToDatabase => "writing nodes to database", -// hannoy::MainStep::DeleteExtraneousTrees => "delete extraneous trees", -// hannoy::MainStep::WriteTheMetadata => "write the metadata", -// } -// .into() -// } - -// fn current(&self) -> u32 { -// *self as u32 -// } - -// fn total(&self) -> u32 { -// Self::CARDINALITY as u32 -// } -// } - -// impl Step for hannoy::SubStep { -// fn name(&self) -> Cow<'static, str> { -// self.unit.into() -// } - -// fn current(&self) -> u32 { -// self.current.load(Ordering::Relaxed) -// } - -// fn total(&self) -> u32 { -// self.max -// } -// } - // Integration with steppe impl steppe::Progress for Progress { @@ -322,7 +281,7 @@ struct Compat(T); impl Step for Compat { fn name(&self) -> Cow<'static, str> { - self.0.name().into() + self.0.name() } fn current(&self) -> u32 { diff --git a/crates/milli/src/update/upgrade/v1_18.rs b/crates/milli/src/update/upgrade/v1_18.rs index e20696b84..ef46e1f5b 100644 --- a/crates/milli/src/update/upgrade/v1_18.rs +++ b/crates/milli/src/update/upgrade/v1_18.rs @@ -14,7 +14,7 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { wtxn: &mut RwTxn, index: &Index, _original: (u32, u32, u32), - _progress: Progress, + progress: Progress, ) -> Result { let embedding_configs = index.embedding_configs(); for config in embedding_configs.embedding_configs(wtxn)? { @@ -22,7 +22,7 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { let quantized = config.config.quantized(); let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); let vector_store = VectorStore::new(index.vector_store, embedder_id, quantized); - vector_store.convert_from_arroy(wtxn)?; + vector_store.convert_from_arroy(wtxn, progress.clone())?; } Ok(false) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 0a97a9bde..181a13e6a 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -155,7 +155,7 @@ impl VectorStore { } } - pub fn convert_from_arroy(&self, wtxn: &mut RwTxn) -> crate::Result<()> { + pub fn convert_from_arroy(&self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { if self.quantized { let dimensions = self .arroy_readers(wtxn, self.arroy_quantized_db()) @@ -168,7 +168,7 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); - let mut builder = writer.builder(&mut rng); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); builder.prepare_arroy_conversion(wtxn)?; builder.build::(wtxn)?; } @@ -186,7 +186,7 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); - let mut builder = writer.builder(&mut rng); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); builder.prepare_arroy_conversion(wtxn)?; builder.build::(wtxn)?; } From f5f2f7c6f201cf3894d215f5cb2302fa07cde97b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 15:09:26 +0200 Subject: [PATCH 20/62] Make the VectorStore aware of the index version --- .../src/scheduler/process_batch.rs | 1 - crates/milli/src/index.rs | 11 ++++-- crates/milli/src/search/new/vector_sort.rs | 7 +++- crates/milli/src/search/similar.rs | 7 +++- .../milli/src/update/index_documents/mod.rs | 11 ++++-- .../src/update/index_documents/transform.rs | 10 ++++-- .../src/update/index_documents/typed_chunk.rs | 8 ++++- crates/milli/src/update/new/indexer/mod.rs | 35 +++++++++++++++---- .../milli/src/update/new/vector_document.rs | 9 +++-- crates/milli/src/update/upgrade/v1_18.rs | 4 ++- crates/milli/src/vector/mod.rs | 8 +++-- 11 files changed, 89 insertions(+), 22 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 4129c57af..efa137cdb 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -147,7 +147,6 @@ impl IndexScheduler { }; let mut index_wtxn = index.write_txn()?; - let index_version = index.get_version(&index_wtxn)?.unwrap_or((1, 12, 0)); let package_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); if index_version != package_version { diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index dc8298a1b..6d02549eb 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1769,10 +1769,12 @@ impl Index { ) -> Result> { let mut res = BTreeMap::new(); let embedders = self.embedding_configs(); + let index_version = self.get_version(rtxn)?.unwrap(); for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); let has_fragments = config.config.embedder_options.has_fragments(); let reader = VectorStore::new( + index_version, self.vector_store, embedder_info.embedder_id, config.config.quantized(), @@ -1795,10 +1797,15 @@ impl Index { pub fn hannoy_stats(&self, rtxn: &RoTxn<'_>) -> Result { let mut stats = HannoyStats::default(); let embedding_configs = self.embedding_configs(); + let index_version = self.get_version(rtxn)?.unwrap(); for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); - let reader = - VectorStore::new(self.vector_store, embedder_id, config.config.quantized()); + let reader = VectorStore::new( + index_version, + self.vector_store, + embedder_id, + config.config.quantized(), + ); reader.aggregate_stats(rtxn, &mut stats)?; } Ok(stats) diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index 284fcd431..fce3340c5 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -56,7 +56,12 @@ impl VectorSort { let target = &self.target; let before = Instant::now(); - let reader = VectorStore::new(ctx.index.vector_store, self.embedder_index, self.quantized); + let reader = VectorStore::new( + ctx.index.get_version(ctx.txn)?.unwrap(), + ctx.index.vector_store, + self.embedder_index, + self.quantized, + ); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 83e65fd6a..d4b45cd4e 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -72,7 +72,12 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let reader = VectorStore::new(self.index.vector_store, embedder_index, self.quantized); + let reader = VectorStore::new( + self.index.get_version(self.rtxn)?.unwrap(), + self.index.vector_store, + embedder_index, + self.quantized, + ); let results = reader.nns_by_item( self.rtxn, self.id, diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index a52cc413c..022146e88 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -485,6 +485,7 @@ where // If an embedder wasn't used in the typedchunk but must be binary quantized // we should insert it in `dimension` + let index_version = self.index.get_version(&self.wtxn)?.unwrap(); for (name, action) in settings_diff.embedding_config_updates.iter() { if action.is_being_quantized && !dimension.contains_key(name.as_str()) { let index = self.index.embedding_configs().embedder_id(self.wtxn, name)?.ok_or( @@ -493,7 +494,12 @@ where key: None, }, )?; - let reader = VectorStore::new(self.index.vector_store, index, action.was_quantized); + let reader = VectorStore::new( + index_version, + self.index.vector_store, + index, + action.was_quantized, + ); let Some(dim) = reader.dimensions(self.wtxn)? else { continue; }; @@ -522,7 +528,8 @@ where let is_quantizing = embedder_config.is_some_and(|action| action.is_being_quantized); pool.install(|| { - let mut writer = VectorStore::new(vector_hannoy, embedder_index, was_quantized); + let mut writer = + VectorStore::new(index_version, vector_hannoy, embedder_index, was_quantized); writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index 985f3a88f..a8e0d318c 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -834,6 +834,7 @@ impl<'a, 'i> Transform<'a, 'i> { None }; + let index_version = self.index.get_version(wtxn)?.unwrap(); let readers: BTreeMap<&str, (VectorStore, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() @@ -842,6 +843,7 @@ impl<'a, 'i> Transform<'a, 'i> { action.write_back() { let reader = VectorStore::new( + index_version, self.index.vector_store, *embedder_id, action.was_quantized, @@ -949,8 +951,12 @@ impl<'a, 'i> Transform<'a, 'i> { else { continue; }; - let hannoy = - VectorStore::new(self.index.vector_store, infos.embedder_id, was_quantized); + let hannoy = VectorStore::new( + index_version, + self.index.vector_store, + infos.embedder_id, + was_quantized, + ); let Some(dimensions) = hannoy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index fe5a8bde8..4efe6bde2 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -619,6 +619,7 @@ pub(crate) fn write_typed_chunk_into_index( let _entered = span.enter(); let embedders = index.embedding_configs(); + let index_version = index.get_version(wtxn)?.unwrap(); let mut remove_vectors_builder = MergerBuilder::new(KeepFirst); let mut manual_vectors_builder = MergerBuilder::new(KeepFirst); @@ -677,7 +678,12 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .is_some_and(|conf| conf.is_quantized); // FIXME: allow customizing distance - let writer = VectorStore::new(index.vector_store, infos.embedder_id, binary_quantized); + let writer = VectorStore::new( + index_version, + index.vector_store, + infos.embedder_id, + binary_quantized, + ); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index b115761cd..71402aae9 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -8,7 +8,7 @@ use document_changes::{DocumentChanges, IndexingContext}; pub use document_deletion::DocumentDeletion; pub use document_operation::{DocumentOperation, PayloadStats}; use hashbrown::HashMap; -use heed::RwTxn; +use heed::{RoTxn, RwTxn}; pub use partial_dump::PartialDump; pub use post_processing::recompute_word_fst_from_word_docids_database; pub use update_by_function::UpdateByFunction; @@ -131,6 +131,7 @@ where let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map); let vector_arroy = index.vector_store; + let index_version = index.get_version(wtxn)?.unwrap(); let hannoy_writers: Result> = embedders .inner_as_ref() .iter() @@ -144,7 +145,12 @@ where })?; let dimensions = runtime.embedder.dimensions(); - let writer = VectorStore::new(vector_arroy, embedder_index, runtime.is_quantized); + let writer = VectorStore::new( + index_version, + vector_arroy, + embedder_index, + runtime.is_quantized, + ); Ok(( embedder_index, @@ -286,6 +292,7 @@ where let index_embedder_category_ids = settings_delta.new_embedder_category_id(); let mut hannoy_writers = hannoy_writers_from_embedder_actions( index, + wtxn, embedder_actions, new_embedders, index_embedder_category_ids, @@ -339,11 +346,13 @@ where fn hannoy_writers_from_embedder_actions<'indexer>( index: &Index, + rtxn: &RoTxn, embedder_actions: &'indexer BTreeMap, embedders: &'indexer RuntimeEmbedders, index_embedder_category_ids: &'indexer std::collections::HashMap, ) -> Result> { let vector_arroy = index.vector_store; + let index_version = index.get_version(rtxn)?.unwrap(); embedders .inner_as_ref() @@ -361,8 +370,12 @@ fn hannoy_writers_from_embedder_actions<'indexer>( }, ))); }; - let writer = - VectorStore::new(vector_arroy, embedder_category_id, action.was_quantized); + let writer = VectorStore::new( + index_version, + vector_arroy, + embedder_category_id, + action.was_quantized, + ); let dimensions = runtime.embedder.dimensions(); Some(Ok(( embedder_category_id, @@ -385,7 +398,12 @@ where let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; }; - let reader = VectorStore::new(index.vector_store, *embedder_id, action.was_quantized); + let reader = VectorStore::new( + index.get_version(wtxn)?.unwrap(), + index.vector_store, + *embedder_id, + action.was_quantized, + ); let Some(dimensions) = reader.dimensions(wtxn)? else { continue; }; @@ -401,7 +419,12 @@ where let Some(infos) = index.embedding_configs().embedder_info(wtxn, embedder_name)? else { continue; }; - let arroy = VectorStore::new(index.vector_store, infos.embedder_id, was_quantized); + let arroy = VectorStore::new( + index.get_version(wtxn)?.unwrap(), + index.vector_store, + infos.embedder_id, + was_quantized, + ); let Some(dimensions) = arroy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index d04f9bb79..76639ad31 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -120,8 +120,13 @@ impl<'t> VectorDocumentFromDb<'t> { config: &IndexEmbeddingConfig, status: &EmbeddingStatus, ) -> Result> { - let reader = - VectorStore::new(self.index.vector_store, embedder_id, config.config.quantized()); + let index_version = self.index.get_version(self.rtxn)?.unwrap(); + let reader = VectorStore::new( + index_version, + self.index.vector_store, + embedder_id, + config.config.quantized(), + ); let vectors = reader.item_vectors(self.rtxn, self.docid)?; Ok(VectorEntry { diff --git a/crates/milli/src/update/upgrade/v1_18.rs b/crates/milli/src/update/upgrade/v1_18.rs index ef46e1f5b..f2e44f0f3 100644 --- a/crates/milli/src/update/upgrade/v1_18.rs +++ b/crates/milli/src/update/upgrade/v1_18.rs @@ -17,11 +17,13 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { progress: Progress, ) -> Result { let embedding_configs = index.embedding_configs(); + let index_version = index.get_version(wtxn)?.unwrap(); for config in embedding_configs.embedding_configs(wtxn)? { // TODO use the embedder name to display progress let quantized = config.config.quantized(); let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); - let vector_store = VectorStore::new(index.vector_store, embedder_id, quantized); + let vector_store = + VectorStore::new(index_version, index.vector_store, embedder_id, quantized); vector_store.convert_from_arroy(wtxn, progress.clone())?; } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 181a13e6a..7a78f15a4 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -47,18 +47,20 @@ const HANNOY_M: usize = 16; const HANNOY_M0: usize = 32; pub struct VectorStore { - quantized: bool, - embedder_index: u8, + version: (u32, u32, u32), database: hannoy::Database, + embedder_index: u8, + quantized: bool, } impl VectorStore { pub fn new( + version: (u32, u32, u32), database: hannoy::Database, embedder_index: u8, quantized: bool, ) -> Self { - Self { database, embedder_index, quantized } + Self { version, database, embedder_index, quantized } } pub fn embedder_index(&self) -> u8 { From fb68f1241c5066a9f4c18d5a09a56ad9919fb852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 16:00:47 +0200 Subject: [PATCH 21/62] Dispatch the vector store based on the index version --- .../src/update/index_documents/transform.rs | 5 +- crates/milli/src/vector/mod.rs | 261 ++++++++++++++---- 2 files changed, 214 insertions(+), 52 deletions(-) diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index a8e0d318c..e0d8b82f8 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -884,10 +884,7 @@ impl<'a, 'i> Transform<'a, 'i> { InternalError::DatabaseMissingEntry { db_name: db_name::DOCUMENTS, key: None }, )?; - let injected_vectors: std::result::Result< - serde_json::Map, - hannoy::Error, - > = readers + let injected_vectors: crate::Result<_> = readers .iter() .filter_map(|(name, (reader, user_provided))| { if !user_provided.contains(docid) { diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 7a78f15a4..8b3ba7577 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -67,6 +67,12 @@ impl VectorStore { self.embedder_index } + /// Whether we must use the arroy to read the vector store. + pub fn version_uses_arroy(&self) -> bool { + let (major, minor, _patch) = self.version; + major == 1 && minor < 18 + } + fn arroy_readers<'a, D: arroy::Distance>( &'a self, rtxn: &'a RoTxn<'a>, @@ -111,14 +117,45 @@ impl VectorStore { rtxn: &RoTxn, store_id: u8, with_items: F, - ) -> Result + ) -> crate::Result where F: FnOnce(&RoaringBitmap) -> O, { - if self.quantized { - self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) + if self.version_uses_arroy() { + if self.quantized { + self._arroy_items_in_store(rtxn, self.arroy_quantized_db(), store_id, with_items) + .map_err(Into::into) + } else { + self._arroy_items_in_store(rtxn, self.arroy_angular_db(), store_id, with_items) + .map_err(Into::into) + } } else { - self._items_in_store(rtxn, self.angular_db(), store_id, with_items) + if self.quantized { + self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) + .map_err(Into::into) + } else { + self._items_in_store(rtxn, self.angular_db(), store_id, with_items) + .map_err(Into::into) + } + } + } + + fn _arroy_items_in_store( + &self, + rtxn: &RoTxn, + db: arroy::Database, + store_id: u8, + with_items: F, + ) -> Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let reader = arroy::Reader::open(rtxn, index, db); + match reader { + Ok(reader) => Ok(with_items(reader.item_ids())), + Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(err) => Err(err), } } @@ -132,7 +169,7 @@ impl VectorStore { where F: FnOnce(&RoaringBitmap) -> O, { - let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let index = vector_store_for_embedder(self.embedder_index, store_id); let reader = hannoy::Reader::open(rtxn, index, db); match reader { Ok(reader) => Ok(with_items(reader.item_ids())), @@ -141,19 +178,35 @@ impl VectorStore { } } - pub fn dimensions(&self, rtxn: &RoTxn) -> Result, hannoy::Error> { - if self.quantized { - Ok(self - .readers(rtxn, self.quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) + pub fn dimensions(&self, rtxn: &RoTxn) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + Ok(self + .arroy_readers(rtxn, self.arroy_quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } else { + Ok(self + .arroy_readers(rtxn, self.arroy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } } else { - Ok(self - .readers(rtxn, self.angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) + if self.quantized { + Ok(self + .readers(rtxn, self.quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } else { + Ok(self + .readers(rtxn, self.angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } } } @@ -336,7 +389,7 @@ impl VectorStore { ) -> Result<(), hannoy::Error> { let dimension = vector.len(); - let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let index = vector_store_for_embedder(self.embedder_index, store_id); let writer = hannoy::Writer::new(db, index, dimension); writer.add_item(wtxn, item_id, vector) } @@ -390,7 +443,7 @@ impl VectorStore { store_id: u8, dimensions: usize, ) -> Result { - let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let index = vector_store_for_embedder(self.embedder_index, store_id); let writer = hannoy::Writer::new(db, index, dimensions); writer.del_item(wtxn, item_id) } @@ -420,7 +473,7 @@ impl VectorStore { store_id: u8, dimensions: usize, ) -> Result<(), hannoy::Error> { - let index = hannoy_store_for_embedder(self.embedder_index, store_id); + let index = vector_store_for_embedder(self.embedder_index, store_id); let writer = hannoy::Writer::new(db, index, dimensions); writer.clear(wtxn) } @@ -481,20 +534,36 @@ impl VectorStore { rtxn: &RoTxn, dimension: usize, item: hannoy::ItemId, - ) -> Result { + ) -> crate::Result { for index in vector_store_range_for_embedder(self.embedder_index) { - let contains = if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; + let contains = if self.version_uses_arroy() { + if self.quantized { + let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + } else { + let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? } - writer.contains_item(rtxn, item)? } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; + if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? } - writer.contains_item(rtxn, item)? }; if contains { return Ok(contains); @@ -509,14 +578,53 @@ impl VectorStore { item: ItemId, limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, hannoy::Error> { - if self.quantized { - self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) + ) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + self._arroy_nns_by_item(rtxn, self.arroy_quantized_db(), item, limit, filter) + .map_err(Into::into) + } else { + self._arroy_nns_by_item(rtxn, self.arroy_angular_db(), item, limit, filter) + .map_err(Into::into) + } } else { - self._nns_by_item(rtxn, self.angular_db(), item, limit, filter) + if self.quantized { + self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) + .map_err(Into::into) + } else { + self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) + } } } + fn _arroy_nns_by_item( + &self, + rtxn: &RoTxn, + db: arroy::Database, + item: ItemId, + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.arroy_readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + if reader.item_ids().is_disjoint(filter) { + continue; + } + searcher.candidates(filter); + } + + if let Some(mut ret) = searcher.by_item(rtxn, item)? { + results.append(&mut ret); + } + } + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + Ok(results) + } + fn _nns_by_item( &self, rtxn: &RoTxn, @@ -552,14 +660,54 @@ impl VectorStore { vector: &[f32], limit: usize, filter: Option<&RoaringBitmap>, - ) -> Result, hannoy::Error> { - if self.quantized { - self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) + ) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + self._arroy_nns_by_vector(rtxn, self.arroy_quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._arroy_nns_by_vector(rtxn, self.arroy_angular_db(), vector, limit, filter) + .map_err(Into::into) + } } else { - self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) + if self.quantized { + self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) + .map_err(Into::into) + } } } + fn _arroy_nns_by_vector( + &self, + rtxn: &RoTxn, + db: arroy::Database, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.arroy_readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + if reader.item_ids().is_disjoint(filter) { + continue; + } + searcher.candidates(filter); + } + + results.append(&mut searcher.by_vector(rtxn, vector)?); + } + + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + + Ok(results) + } + fn _nns_by_vector( &self, rtxn: &RoTxn, @@ -589,22 +737,39 @@ impl VectorStore { Ok(results) } - pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> Result>, hannoy::Error> { + pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { let mut vectors = Vec::new(); - if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); + if self.version_uses_arroy() { + if self.quantized { + for reader in self.arroy_readers(rtxn, self.arroy_quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self.arroy_readers(rtxn, self.arroy_angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } } } } else { - for reader in self.readers(rtxn, self.angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); + if self.quantized { + for reader in self.readers(rtxn, self.quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self.readers(rtxn, self.angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } } } } + Ok(vectors) } @@ -1300,10 +1465,10 @@ pub const fn is_cuda_enabled() -> bool { } fn vector_store_range_for_embedder(embedder_id: u8) -> impl Iterator { - (0..=u8::MAX).map(move |store_id| hannoy_store_for_embedder(embedder_id, store_id)) + (0..=u8::MAX).map(move |store_id| vector_store_for_embedder(embedder_id, store_id)) } -fn hannoy_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { +fn vector_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { let embedder_id = (embedder_id as u16) << 8; embedder_id | (store_id as u16) } From 4645813ea85eccfe8b446072d1f11998f3cb7a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 16:20:32 +0200 Subject: [PATCH 22/62] Make clippy happy --- crates/milli/src/progress.rs | 1 - .../milli/src/update/index_documents/mod.rs | 2 +- crates/milli/src/vector/mod.rs | 102 ++++++++---------- 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index 728e95652..eb309b0b0 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -5,7 +5,6 @@ use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; -use enum_iterator::Sequence; use indexmap::IndexMap; use itertools::Itertools; use serde::Serialize; diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 022146e88..465a40f42 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -485,7 +485,7 @@ where // If an embedder wasn't used in the typedchunk but must be binary quantized // we should insert it in `dimension` - let index_version = self.index.get_version(&self.wtxn)?.unwrap(); + let index_version = self.index.get_version(self.wtxn)?.unwrap(); for (name, action) in settings_diff.embedding_config_updates.iter() { if action.is_being_quantized && !dimension.contains_key(name.as_str()) { let index = self.index.embedding_configs().embedder_id(self.wtxn, name)?.ok_or( diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 8b3ba7577..bf4ae7ba4 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -129,14 +129,12 @@ impl VectorStore { self._arroy_items_in_store(rtxn, self.arroy_angular_db(), store_id, with_items) .map_err(Into::into) } + } else if self.quantized { + self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) + .map_err(Into::into) } else { - if self.quantized { - self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) - .map_err(Into::into) - } else { - self._items_in_store(rtxn, self.angular_db(), store_id, with_items) - .map_err(Into::into) - } + self._items_in_store(rtxn, self.angular_db(), store_id, with_items) + .map_err(Into::into) } } @@ -193,20 +191,18 @@ impl VectorStore { .transpose()? .map(|reader| reader.dimensions())) } + } else if self.quantized { + Ok(self + .readers(rtxn, self.quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) } else { - if self.quantized { - Ok(self - .readers(rtxn, self.quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } else { - Ok(self - .readers(rtxn, self.angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } + Ok(self + .readers(rtxn, self.angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) } } @@ -550,20 +546,18 @@ impl VectorStore { } writer.contains_item(rtxn, item)? } - } else { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? - } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? + } else if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; } + writer.contains_item(rtxn, item)? + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? }; if contains { return Ok(contains); @@ -587,13 +581,11 @@ impl VectorStore { self._arroy_nns_by_item(rtxn, self.arroy_angular_db(), item, limit, filter) .map_err(Into::into) } + } else if self.quantized { + self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) + .map_err(Into::into) } else { - if self.quantized { - self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) - .map_err(Into::into) - } else { - self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) - } + self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) } } @@ -669,14 +661,12 @@ impl VectorStore { self._arroy_nns_by_vector(rtxn, self.arroy_angular_db(), vector, limit, filter) .map_err(Into::into) } + } else if self.quantized { + self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) + .map_err(Into::into) } else { - if self.quantized { - self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) - .map_err(Into::into) - } else { - self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) - .map_err(Into::into) - } + self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) + .map_err(Into::into) } } @@ -754,18 +744,16 @@ impl VectorStore { } } } - } else { - if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } + } else if self.quantized { + for reader in self.readers(rtxn, self.quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); } - } else { - for reader in self.readers(rtxn, self.angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } + } + } else { + for reader in self.readers(rtxn, self.angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); } } } From 97ea9e993701c6e1adce2f462c6ff3ec4f6c66b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 16:25:51 +0200 Subject: [PATCH 23/62] Make cargo fmt happy --- crates/milli/src/documents/geo_sort.rs | 20 ++++++++------------ crates/milli/src/documents/sort.rs | 19 ++++++++----------- crates/milli/src/update/indexer_config.rs | 3 ++- crates/milli/src/vector/mod.rs | 9 +++------ 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/crates/milli/src/documents/geo_sort.rs b/crates/milli/src/documents/geo_sort.rs index 0750dfe5c..8e574ec7c 100644 --- a/crates/milli/src/documents/geo_sort.rs +++ b/crates/milli/src/documents/geo_sort.rs @@ -1,17 +1,13 @@ -use crate::{ - distance_between_two_points, - heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}, - lat_lng_to_xyz, - search::new::{facet_string_values, facet_values_prefix_key}, - GeoPoint, Index, -}; -use heed::{ - types::{Bytes, Unit}, - RoPrefix, RoTxn, -}; +use std::collections::VecDeque; + +use heed::types::{Bytes, Unit}; +use heed::{RoPrefix, RoTxn}; use roaring::RoaringBitmap; use rstar::RTree; -use std::collections::VecDeque; + +use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; +use crate::search::new::{facet_string_values, facet_values_prefix_key}; +use crate::{distance_between_two_points, lat_lng_to_xyz, GeoPoint, Index}; #[derive(Debug, Clone, Copy)] pub struct GeoSortParameter { diff --git a/crates/milli/src/documents/sort.rs b/crates/milli/src/documents/sort.rs index 3866d9e27..f76081847 100644 --- a/crates/milli/src/documents/sort.rs +++ b/crates/milli/src/documents/sort.rs @@ -1,19 +1,16 @@ use std::collections::{BTreeSet, VecDeque}; -use crate::{ - constants::RESERVED_GEO_FIELD_NAME, - documents::{geo_sort::next_bucket, GeoSortParameter}, - heed_codec::{ - facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, - BytesRefCodec, - }, - is_faceted, - search::facet::{ascending_facet_sort, descending_facet_sort}, - AscDesc, DocumentId, Member, UserError, -}; use heed::Database; use roaring::RoaringBitmap; +use crate::constants::RESERVED_GEO_FIELD_NAME; +use crate::documents::geo_sort::next_bucket; +use crate::documents::GeoSortParameter; +use crate::heed_codec::facet::{FacetGroupKeyCodec, FacetGroupValueCodec}; +use crate::heed_codec::BytesRefCodec; +use crate::search::facet::{ascending_facet_sort, descending_facet_sort}; +use crate::{is_faceted, AscDesc, DocumentId, Member, UserError}; + #[derive(Debug, Clone, Copy)] enum AscDescId { Facet { field_id: u16, ascending: bool }, diff --git a/crates/milli/src/update/indexer_config.rs b/crates/milli/src/update/indexer_config.rs index 845da5a51..39a013d13 100644 --- a/crates/milli/src/update/indexer_config.rs +++ b/crates/milli/src/update/indexer_config.rs @@ -1,7 +1,8 @@ use grenad::CompressionType; use super::GrenadParameters; -use crate::{thread_pool_no_abort::ThreadPoolNoAbort, ThreadPoolNoAbortBuilder}; +use crate::thread_pool_no_abort::ThreadPoolNoAbort; +use crate::ThreadPoolNoAbortBuilder; #[derive(Debug)] pub struct IndexerConfig { diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index bf4ae7ba4..f93540614 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -133,8 +133,7 @@ impl VectorStore { self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) .map_err(Into::into) } else { - self._items_in_store(rtxn, self.angular_db(), store_id, with_items) - .map_err(Into::into) + self._items_in_store(rtxn, self.angular_db(), store_id, with_items).map_err(Into::into) } } @@ -582,8 +581,7 @@ impl VectorStore { .map_err(Into::into) } } else if self.quantized { - self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter) - .map_err(Into::into) + self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter).map_err(Into::into) } else { self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) } @@ -665,8 +663,7 @@ impl VectorStore { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) .map_err(Into::into) } else { - self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter) - .map_err(Into::into) + self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter).map_err(Into::into) } } From 72c63d3929790aab72a0e6e202ee1fc3d6b1f47d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 12 Aug 2025 16:03:00 +0200 Subject: [PATCH 24/62] Move code to the right file --- crates/milli/src/update/upgrade/mod.rs | 1 + crates/milli/src/update/upgrade/v1_17.rs | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 crates/milli/src/update/upgrade/v1_17.rs diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 40826cbe1..8f6a74b00 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -3,6 +3,7 @@ mod v1_13; mod v1_14; mod v1_15; mod v1_16; +mod v1_17; mod v1_18; use heed::RwTxn; diff --git a/crates/milli/src/update/upgrade/v1_17.rs b/crates/milli/src/update/upgrade/v1_17.rs new file mode 100644 index 000000000..412aa21d8 --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_17.rs @@ -0,0 +1,24 @@ +use heed::RwTxn; + +use super::UpgradeIndex; +use crate::progress::Progress; +use crate::{Index, Result}; + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_16_To_V1_17_0(); + +impl UpgradeIndex for Latest_V1_16_To_V1_17_0 { + fn upgrade( + &self, + _wtxn: &mut RwTxn, + _index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 17, 0) + } +} From 5139dd273e241e7d486b69218f6c46ccf1adbebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 12 Aug 2025 18:00:54 +0200 Subject: [PATCH 25/62] Depend on Hannoy from crates.io --- Cargo.lock | 7 ++++--- crates/milli/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54531907a..ee00d3db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2603,8 +2603,9 @@ dependencies = [ [[package]] name = "hannoy" -version = "0.0.2" -source = "git+https://github.com/nnethercott/hannoy?branch=main#8d1846b188ed2cc8776fdb86805eefbfbde9ddd1" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cac6ebc04fc7246356d29908b55315c26c695a2ea2f692de9f72c0ac61ca1b1" dependencies = [ "bytemuck", "byteorder", @@ -3363,7 +3364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 009e874fa..549fdcdc7 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.1" -hannoy = { git = "https://github.com/nnethercott/hannoy", branch = "main" } +hannoy = "0.0.3" rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } From f4147a60a3d24f84f8e166629a9cffed977bcd0f Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 13 Aug 2025 12:58:12 +0200 Subject: [PATCH 26/62] Remove the vector_store reference --- crates/milli/src/index.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 6d02549eb..b9210fa60 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -237,7 +237,7 @@ impl Index { // vector stuff let embedder_category_id = env.create_database(&mut wtxn, Some(VECTOR_EMBEDDER_CATEGORY_ID))?; - let vector_hannoy = env.create_database(&mut wtxn, Some(VECTOR_STORE))?; + let vector_store = env.create_database(&mut wtxn, Some(VECTOR_STORE))?; let documents = env.create_database(&mut wtxn, Some(DOCUMENTS))?; @@ -264,7 +264,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_store: vector_hannoy, + vector_store, embedder_category_id, documents, }; From a8e9597f49080857e0174a9cdf66e884b5fefefc Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 13 Aug 2025 13:52:01 +0200 Subject: [PATCH 27/62] Make cargo insta happy --- crates/meilisearch/tests/vector/binary_quantized.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index c5a7c9244..ec82290ff 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -104,8 +104,8 @@ async fn binary_quantize_before_sending_documents() { "manual": { "embeddings": [ [ - -1.0, - -1.0, + 0.0, + 0.0, 1.0 ] ], @@ -122,7 +122,7 @@ async fn binary_quantize_before_sending_documents() { [ 1.0, 1.0, - -1.0 + 0.0 ] ], "regenerate": false @@ -191,8 +191,8 @@ async fn binary_quantize_after_sending_documents() { "manual": { "embeddings": [ [ - -1.0, - -1.0, + 0.0, + 0.0, 1.0 ] ], @@ -209,7 +209,7 @@ async fn binary_quantize_after_sending_documents() { [ 1.0, 1.0, - -1.0 + 0.0 ] ], "regenerate": false From 6d9e0c4bce909a72b47d860fab5c927db0db78a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 14 Aug 2025 10:57:33 +0200 Subject: [PATCH 28/62] Switch to hannoy 0.0.4 --- Cargo.lock | 832 ++++++++++++++++++++------------- crates/milli/Cargo.toml | 2 +- crates/milli/src/vector/mod.rs | 6 - 3 files changed, 505 insertions(+), 335 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee00d3db4..17f3b3daf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,7 +55,7 @@ dependencies = [ "flate2", "foldhash", "futures-core", - "h2 0.3.26", + "h2 0.3.27", "http 0.2.12", "httparse", "httpdate", @@ -65,7 +65,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.9.1", + "rand 0.9.2", "sha1", "smallvec", "tokio", @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -121,7 +121,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2", + "socket2 0.5.10", "tokio", "tracing", ] @@ -202,7 +202,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.5.10", "time", "tracing", "url", @@ -217,14 +217,14 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] name = "actix-web-lab" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33034dd88446a5deb20e42156dbfe43d07e0499345db3ae65b3f51854190531" +checksum = "781b41742426e73ab2fe531a7d204b9ffa0815a63e358bcd24f2c3c69cda644a" dependencies = [ "actix-http", "actix-router", @@ -268,9 +268,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "adler32" @@ -358,9 +358,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -373,44 +373,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" dependencies = [ "backtrace", ] @@ -460,7 +460,7 @@ dependencies = [ "rayon", "roaring", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "tracing", ] @@ -492,7 +492,7 @@ dependencies = [ "secrecy", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-stream", "tokio-util", @@ -506,7 +506,7 @@ source = "git+https://github.com/meilisearch/async-openai?branch=better-error-ha dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -517,7 +517,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -528,9 +528,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "backoff" @@ -624,6 +624,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.70.1" @@ -639,7 +659,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -733,7 +753,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -779,9 +799,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.18.1" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" dependencies = [ "allocator-api2 0.2.21", "serde", @@ -796,7 +816,7 @@ dependencies = [ "allocator-api2 0.2.21", "bitpacking", "bumpalo", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "serde", "serde_json", ] @@ -842,22 +862,22 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.23.1" +version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.9.3" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1" +checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -883,28 +903,18 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] name = "camino" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da45bc31171d8d6960122e222a67740df867c1dd53b4d51caa297084c185cab" +checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" dependencies = [ "serde", ] @@ -923,7 +933,7 @@ dependencies = [ "memmap2", "num-traits", "num_cpus", - "rand 0.9.1", + "rand 0.9.2", "rand_distr", "rayon", "safetensors", @@ -969,7 +979,7 @@ dependencies = [ "candle-nn", "fancy-regex", "num-traits", - "rand 0.9.1", + "rand 0.9.2", "rayon", "serde", "serde_json", @@ -997,7 +1007,7 @@ dependencies = [ "serde-untagged", "serde-value", "thiserror 1.0.69", - "toml", + "toml 0.8.23", "unicode-xid", "url", ] @@ -1014,17 +1024,17 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] name = "cargo_toml" -version = "0.22.1" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02260d489095346e5cafd04dea8e8cb54d1d74fcd759022a9b72986ebe9a1257" +checksum = "374b7c592d9c00c1f4972ea58390ac6b18cbb6ab79011f3bdc90a0b82ca06b77" dependencies = [ "serde", - "toml", + "toml 0.9.5", ] [[package]] @@ -1035,9 +1045,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.25" +version = "1.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" +checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" dependencies = [ "jobserver", "libc", @@ -1064,9 +1074,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -1156,9 +1166,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", "clap_derive", @@ -1166,9 +1176,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -1178,21 +1188,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "color-spantrace" @@ -1208,9 +1218,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "concat-arrays" @@ -1327,9 +1337,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] @@ -1412,9 +1422,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" @@ -1449,9 +1459,9 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.16.4" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9574894139a982bf26fbb44473a9d416c015e779c51ef0fbc0789f1a1c17b25" +checksum = "17200eb07e7d85a243aa1bf4569a7aa998385ba98d14833973a817a63cc86e92" dependencies = [ "half", "libloading", @@ -1502,7 +1512,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1524,7 +1534,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1584,7 +1594,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1626,7 +1636,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1646,7 +1656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1666,7 +1676,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "unicode-xid", ] @@ -1696,7 +1706,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1760,7 +1770,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1790,7 +1800,7 @@ dependencies = [ "serde_json", "tar", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", "tracing", "uuid", @@ -1921,7 +1931,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1941,7 +1951,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -1962,12 +1972,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2009,7 +2019,7 @@ name = "file-store" version = "1.19.0" dependencies = [ "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "tracing", "uuid", ] @@ -2154,7 +2164,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -2493,7 +2503,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2532,9 +2542,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "grenad" @@ -2551,9 +2561,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" dependencies = [ "bytes", "fnv", @@ -2570,9 +2580,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ "atomic-waker", "bytes", @@ -2597,19 +2607,19 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.1", + "rand 0.9.2", "rand_distr", ] [[package]] name = "hannoy" -version = "0.0.3" +version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cac6ebc04fc7246356d29908b55315c26c695a2ea2f692de9f72c0ac61ca1b1" +checksum = "a80496a4713fefbf4ea388b30288afaa23566173510a694d838e0a85a3ada5c0" dependencies = [ "bytemuck", "byteorder", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "heed", "min-max-heap", "page_size", @@ -2619,7 +2629,7 @@ dependencies = [ "roaring", "rustc-hash 2.1.1", "steppe", - "thiserror 2.0.12", + "thiserror 2.0.14", "tinyvec", "tracing", ] @@ -2654,9 +2664,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2 0.2.21", "equivalent", @@ -2710,7 +2720,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" dependencies = [ - "bincode", + "bincode 1.3.3", "byteorder", "heed-traits", "serde", @@ -2719,9 +2729,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2820,7 +2830,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.10", + "h2 0.4.12", "http 1.3.1", "http-body", "httparse", @@ -2834,9 +2844,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.6" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", "hyper", @@ -2847,14 +2857,14 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.0", + "webpki-roots 1.0.2", ] [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ "base64 0.22.1", "bytes", @@ -2868,7 +2878,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -3013,7 +3023,7 @@ dependencies = [ "libflate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -3023,7 +3033,7 @@ dependencies = [ "anyhow", "backoff", "big_s", - "bincode", + "bincode 1.3.3", "bumpalo", "bumparaw-collections", "byte-unit", @@ -3049,7 +3059,7 @@ dependencies = [ "serde_json", "synchronoise", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", "tracing", "ureq", @@ -3063,7 +3073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "serde", ] @@ -3113,6 +3123,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -3219,9 +3240,9 @@ dependencies = [ [[package]] name = "jieba-rs" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06096b4b61fb4bfdbf16c6a968ea2d6be1ac9617cf3db741c3b641e6c290a35" +checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a" dependencies = [ "cedarwood", "fxhash", @@ -3316,10 +3337,16 @@ dependencies = [ ] [[package]] -name = "libc" -version = "0.2.172" +name = "libbz2-rs-sys" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libflate" @@ -3347,9 +3374,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.18.1+1.9.0" +version = "0.18.2+1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1dcb20f84ffcdd825c7a311ae347cce604a6f084a767dec4a4929829645290e" +checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" dependencies = [ "cc", "libc", @@ -3364,14 +3391,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.3", ] [[package]] name = "liblzma" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +checksum = "0791ab7e08ccc8e0ce893f6906eb2703ed8739d8e89b57c0714e71bad09024c8" dependencies = [ "liblzma-sys", ] @@ -3416,9 +3443,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ "bitflags 2.9.1", "libc", @@ -3448,12 +3475,12 @@ dependencies = [ [[package]] name = "lindera" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f20720cb4206e87b6844b05c66b23301e7bb532718f200ff55bbbdfbce9b7f2b" +checksum = "877750979d709bb5fb2d616a2c9968301ead80147db2270c8f23d8239467f159" dependencies = [ "anyhow", - "bincode", + "bincode 2.0.1", "byteorder", "csv", "kanaria", @@ -3478,11 +3505,11 @@ dependencies = [ [[package]] name = "lindera-cc-cedict" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6ddd4aeaeaf1ce47ea5785bd6a273179d32df4af4b306d9b65a7a7f81a0e61" +checksum = "4efb27e037efbd41fdce30e3771084a46040cdfdbec718425879f8aa7dfa16f2" dependencies = [ - "bincode", + "bincode 2.0.1", "byteorder", "lindera-dictionary", "once_cell", @@ -3491,12 +3518,12 @@ dependencies = [ [[package]] name = "lindera-dictionary" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b5e417c4c6e001459e019b178f65f759be9c2cbf2d9bd803ec5d8ed0e62124" +checksum = "4cbb45e81527092e1af24c45bf068376df34faaa9f98d44a99ec59c7edfdb45a" dependencies = [ "anyhow", - "bincode", + "bincode 2.0.1", "byteorder", "csv", "derive_builder 0.20.2", @@ -3508,22 +3535,22 @@ dependencies = [ "log", "md5", "once_cell", - "rand 0.9.1", + "rand 0.9.2", "reqwest", "serde", "tar", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "yada", ] [[package]] name = "lindera-ipadic" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2867975f1b92d1093ccbb52c5c1664a56dfbd27a2fece0166c765ad1f043f31" +checksum = "447887ebb06c9faf7e9b41c03b491ae3e12aca2c0e119a865b682863a1b538aa" dependencies = [ - "bincode", + "bincode 2.0.1", "byteorder", "lindera-dictionary", "once_cell", @@ -3532,11 +3559,11 @@ dependencies = [ [[package]] name = "lindera-ipadic-neologd" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54c4c2d3fb8b380d0ace5ae97111ca444bcfa7721966f552117d57f07d8b3b1" +checksum = "8da7829c17dee5b1068f241c1989f83a2d98307ed5d3b87fcc91bd06b7a7b44e" dependencies = [ - "bincode", + "bincode 2.0.1", "byteorder", "lindera-dictionary", "once_cell", @@ -3545,11 +3572,11 @@ dependencies = [ [[package]] name = "lindera-ko-dic" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f495e64f62deee60d9b71dbe3fd39b69b8688c9d591842f81f94e200eb4d81f" +checksum = "8dc88354c6f8fbdeb7aa3d44673eb0ca03b7ad6e8d4edb62332604bb7806d17b" dependencies = [ - "bincode", + "bincode 2.0.1", "byteorder", "lindera-dictionary", "once_cell", @@ -3558,11 +3585,11 @@ dependencies = [ [[package]] name = "lindera-unidic" -version = "0.43.1" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85ff97ce04c519fbca0f05504ea028761ccc456b1e84cf1e75fac57f9b3caf1" +checksum = "4cf374587e0202193c73f443cb0ba0d34ff0013949b2355aeeeb0f840679681e" dependencies = [ - "bincode", + "bincode 2.0.1", "byteorder", "lindera-dictionary", "once_cell", @@ -3624,7 +3651,7 @@ checksum = "de66c928222984aea59fcaed8ba627f388aaac3c1f57dcb05cc25495ef8faefe" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -3697,7 +3724,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" dependencies = [ - "hashbrown 0.15.4", + "hashbrown 0.15.5", ] [[package]] @@ -3731,7 +3758,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -3836,10 +3863,10 @@ dependencies = [ "temp-env", "tempfile", "termcolor", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", "tokio", - "toml", + "toml 0.8.23", "tracing", "tracing-actix-web", "tracing-subscriber", @@ -3851,7 +3878,7 @@ dependencies = [ "uuid", "wiremock", "yaup", - "zip 4.1.0", + "zip 4.3.0", ] [[package]] @@ -3868,7 +3895,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", "uuid", ] @@ -3901,7 +3928,7 @@ dependencies = [ "serde_json", "tar", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", "tokio", "utoipa", @@ -3951,7 +3978,7 @@ dependencies = [ "bbqueue", "big_s", "bimap", - "bincode", + "bincode 1.3.3", "bstr", "bumpalo", "bumparaw-collections", @@ -3976,7 +4003,7 @@ dependencies = [ "geoutils", "grenad", "hannoy", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "heed", "hf-hub", "indexmap", @@ -4009,7 +4036,7 @@ dependencies = [ "smartstring", "steppe", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.14", "thread_local", "tiktoken-rs", "time", @@ -4061,9 +4088,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] @@ -4076,7 +4103,7 @@ checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] @@ -4098,7 +4125,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -4261,23 +4288,24 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" dependencies = [ "num_enum_derive", + "rustversion", ] [[package]] name = "num_enum_derive" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -4430,9 +4458,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owo-colors" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" [[package]] name = "page_size" @@ -4534,20 +4562,20 @@ dependencies = [ [[package]] name = "pest" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" +checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.14", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" +checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" dependencies = [ "pest", "pest_generator", @@ -4555,24 +4583,23 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" +checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] name = "pest_meta" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" +checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" dependencies = [ - "once_cell", "pest", "sha2", ] @@ -4617,7 +4644,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -4646,7 +4673,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -4712,9 +4739,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" @@ -4731,6 +4758,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4751,9 +4784,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" dependencies = [ "unicode-ident", ] @@ -4794,7 +4827,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -4876,8 +4909,8 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2", - "thiserror 2.0.12", + "socket2 0.5.10", + "thiserror 2.0.14", "tokio", "tracing", "web-time", @@ -4892,13 +4925,13 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand 0.9.1", + "rand 0.9.2", "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.14", "tinyvec", "tracing", "web-time", @@ -4906,14 +4939,14 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.5.10", "tracing", "windows-sys 0.59.0", ] @@ -4929,9 +4962,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" @@ -4952,9 +4985,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -5005,7 +5038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.1", + "rand 0.9.2", ] [[package]] @@ -5028,9 +5061,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -5049,9 +5082,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -5065,9 +5098,9 @@ checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" [[package]] name = "redox_syscall" -version = "0.5.12" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ "bitflags 2.9.1", ] @@ -5129,9 +5162,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.20" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ "base64 0.22.1", "bytes", @@ -5168,7 +5201,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.0", + "webpki-roots 1.0.2", ] [[package]] @@ -5214,7 +5247,7 @@ checksum = "a5a11a05ee1ce44058fa3d5961d05194fdbe3ad6b40f904af764d81b86450e6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -5291,9 +5324,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.37.1" +version = "1.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7de2ba56ac291bd90c6b9bece784a52ae1411f9506544b3eae36dd2356d50" +checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" dependencies = [ "arrayvec", "borsh", @@ -5307,9 +5340,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustc-hash" @@ -5338,22 +5371,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "log", "once_cell", @@ -5397,9 +5430,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "ring", "rustls-pki-types", @@ -5408,9 +5441,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -5470,9 +5503,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" dependencies = [ "bitflags 2.9.1", "core-foundation", @@ -5501,7 +5534,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", ] @@ -5550,9 +5583,9 @@ dependencies = [ [[package]] name = "serde-untagged" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299d9c19d7d466db4ab10addd5703e4c615dec2a5a16dbbafe191045e87ee66e" +checksum = "34836a629bcbc6f1afdf0907a744870039b1e14c0561cb26094fa683b158eff3" dependencies = [ "erased-serde", "serde", @@ -5577,7 +5610,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -5595,9 +5628,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "indexmap", "itoa", @@ -5634,6 +5667,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5709,9 +5751,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" dependencies = [ "libc", ] @@ -5742,7 +5784,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", ] @@ -5754,12 +5796,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "slice-group-by" @@ -5769,9 +5808,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallstr" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +checksum = "862077b1e764f04c251fe82a2ef562fd78d7cadaeb072ca7c2bcaf7217b1ff3b" dependencies = [ "serde", "smallvec", @@ -5808,6 +5847,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "socks" version = "0.3.4" @@ -5893,24 +5942,23 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck", "proc-macro2", "quote", - "rustversion", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -5932,9 +5980,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" dependencies = [ "proc-macro2", "quote", @@ -5967,7 +6015,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -6047,7 +6095,7 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix 1.0.7", + "rustix 1.0.8", "windows-sys 0.59.0", ] @@ -6080,11 +6128,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.14", ] [[package]] @@ -6095,18 +6143,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -6242,20 +6290,22 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6266,7 +6316,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -6292,9 +6342,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", @@ -6310,11 +6360,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit", ] +[[package]] +name = "toml" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 1.0.0", + "toml_datetime 0.7.0", + "toml_parser", + "toml_writer", + "winnow", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -6324,6 +6389,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -6332,18 +6406,33 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", "winnow", ] +[[package]] +name = "toml_parser" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +dependencies = [ + "winnow", +] + [[package]] name = "toml_write" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" + [[package]] name = "tower" version = "0.5.2" @@ -6403,9 +6492,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2340b7722695166c7fc9b3e3cd1166e7c74fedb9075b8f0c74d3822d2e41caf5" +checksum = "5360edd490ec8dee9fedfc6a9fd83ac2f01b3e1996e3261b9ad18a61971fe064" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -6416,20 +6505,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -6569,7 +6658,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c01d12e3a56a4432a8b436f293c25f4808bdf9e9f9f98f9260bba1f1bc5a1f26" dependencies = [ - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -6616,9 +6705,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" @@ -6644,6 +6733,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.12.1" @@ -6720,7 +6815,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.101", + "syn 2.0.105", "uuid", ] @@ -6738,9 +6833,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom 0.3.3", "js-sys", @@ -6804,6 +6899,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "walkdir" version = "2.5.0" @@ -6836,9 +6937,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -6871,7 +6972,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "wasm-bindgen-shared", ] @@ -6906,7 +7007,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6959,14 +7060,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.0", + "webpki-roots 1.0.2", ] [[package]] name = "webpki-roots" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] @@ -7066,7 +7167,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -7077,7 +7178,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -7141,6 +7242,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -7165,13 +7275,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows-threading" version = "0.1.0" @@ -7193,6 +7320,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -7205,6 +7338,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -7217,12 +7356,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -7235,6 +7386,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -7247,6 +7404,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -7259,6 +7422,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -7272,19 +7441,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -version = "0.7.10" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" dependencies = [ "memchr", ] [[package]] name = "wiremock" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "101681b74cd87b5899e87bcf5a64e83334dd313fcd3053ea72e6dba18928e301" +checksum = "a2b8b99d4cdbf36b239a9532e31fe4fb8acc38d1897c1761e161550a7dc78e6a" dependencies = [ "assert-json-diff", "async-trait", @@ -7330,12 +7505,12 @@ dependencies = [ [[package]] name = "xattr" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" +checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" dependencies = [ "libc", - "rustix 1.0.7", + "rustix 1.0.8", ] [[package]] @@ -7410,7 +7585,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "synstructure", ] @@ -7422,28 +7597,28 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -7463,7 +7638,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", "synstructure", ] @@ -7484,7 +7659,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -7500,9 +7675,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke 0.8.0", "zerofrom", @@ -7517,7 +7692,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.105", ] [[package]] @@ -7537,9 +7712,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.1.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7dcdb4229c0e79c2531a24de7726a0e980417a74fb4d030a35f535665439a0" +checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" dependencies = [ "aes", "arbitrary", @@ -7554,6 +7729,7 @@ dependencies = [ "liblzma", "memchr", "pbkdf2", + "ppmd-rust", "sha1", "time", "zeroize", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 549fdcdc7..e90d96500 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.1" -hannoy = "0.0.3" +hannoy = "0.0.4" rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f93540614..9487cd9b1 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -630,9 +630,6 @@ impl VectorStore { let mut searcher = reader.nns(limit); searcher.ef_search((limit * 10).max(100)); // TODO find better ef if let Some(filter) = filter { - if reader.item_ids().is_disjoint(filter) { - continue; - } searcher.candidates(filter); } @@ -710,9 +707,6 @@ impl VectorStore { let mut searcher = reader.nns(limit); searcher.ef_search((limit * 10).max(100)); // TODO find better ef if let Some(filter) = filter { - if reader.item_ids().is_disjoint(filter) { - continue; - } searcher.candidates(filter); } From 31cb960992043d5a7a55fdaabd7b7c6fd01d7270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 14 Aug 2025 11:47:05 +0200 Subject: [PATCH 29/62] Make clippy happy --- .../milli/src/search/facet/filter_vector.rs | 22 +++++---- .../extract/extract_word_docids.rs | 47 +------------------ .../milli/src/update/new/extract/documents.rs | 4 +- crates/milli/src/update/new/merger.rs | 8 ++-- crates/tracing-trace/src/main.rs | 2 +- 5 files changed, 21 insertions(+), 62 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 1ef4b8e3d..62303c622 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -3,7 +3,7 @@ use roaring::{MultiOps, RoaringBitmap}; use crate::error::{DidYouMean, Error}; use crate::vector::db::IndexEmbeddingConfig; -use crate::vector::{ArroyStats, ArroyWrapper}; +use crate::vector::{HannoyStats, VectorStore}; use crate::Index; #[derive(Debug, thiserror::Error)] @@ -82,6 +82,7 @@ fn evaluate_inner( embedding_configs: &[IndexEmbeddingConfig], filter: &VectorFilter<'_>, ) -> crate::Result { + let index_version = index.get_version(rtxn)?.unwrap(); let embedder_name = embedder.value(); let available_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); @@ -96,8 +97,9 @@ fn evaluate_inner( .embedder_info(rtxn, embedder_name)? .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, + let vector_store = VectorStore::new( + index_version, + index.vector_store, embedder_info.embedder_id, embedding_config.config.quantized(), ); @@ -122,7 +124,7 @@ fn evaluate_inner( })?; let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| { + vector_store.items_in_store(rtxn, fragment_config.id, |bitmap| { bitmap.clone() - user_provided_docids })? } @@ -132,8 +134,8 @@ fn evaluate_inner( } let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let mut stats = HannoyStats::default(); + vector_store.aggregate_stats(rtxn, &mut stats)?; stats.documents - user_provided_docids.clone() } VectorFilter::UserProvided => { @@ -141,14 +143,14 @@ fn evaluate_inner( user_provided_docids.clone() } VectorFilter::Regenerate => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let mut stats = HannoyStats::default(); + vector_store.aggregate_stats(rtxn, &mut stats)?; let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); stats.documents - skip_regenerate } VectorFilter::None => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let mut stats = HannoyStats::default(); + vector_store.aggregate_stats(rtxn, &mut stats)?; stats.documents } }; diff --git a/crates/milli/src/update/index_documents/extract/extract_word_docids.rs b/crates/milli/src/update/index_documents/extract/extract_word_docids.rs index a964c0bbe..6d28adb2b 100644 --- a/crates/milli/src/update/index_documents/extract/extract_word_docids.rs +++ b/crates/milli/src/update/index_documents/extract/extract_word_docids.rs @@ -2,9 +2,8 @@ use std::collections::BTreeSet; use std::fs::File; use std::io::{self, BufReader}; -use heed::{BytesDecode, BytesEncode}; +use heed::BytesDecode; use obkv::KvReaderU16; -use roaring::RoaringBitmap; use super::helpers::{ create_sorter, create_writer, try_split_array_at, writer_into_reader, GrenadParameters, @@ -16,7 +15,7 @@ use crate::index::db_name::DOCID_WORD_POSITIONS; use crate::update::del_add::{is_noop_del_add_obkv, DelAdd, KvReaderDelAdd, KvWriterDelAdd}; use crate::update::index_documents::helpers::sorter_into_reader; use crate::update::settings::InnerIndexSettingsDiff; -use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result}; +use crate::{DocumentId, FieldId, Result}; /// Extracts the word and the documents ids where this word appear. /// @@ -201,45 +200,3 @@ fn words_into_sorter( Ok(()) } - -#[tracing::instrument(level = "trace", skip_all, target = "indexing::extract")] -fn docids_into_writers( - word: &str, - deletions: &RoaringBitmap, - additions: &RoaringBitmap, - writer: &mut grenad::Writer, -) -> Result<()> -where - W: std::io::Write, -{ - if deletions == additions { - // if the same value is deleted and added, do nothing. - return Ok(()); - } - - // Write each value in the same KvDelAdd before inserting it in the final writer. - let mut obkv = KvWriterDelAdd::memory(); - // deletions: - if !deletions.is_empty() && !deletions.is_subset(additions) { - obkv.insert( - DelAdd::Deletion, - CboRoaringBitmapCodec::bytes_encode(deletions).map_err(|_| { - SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) } - })?, - )?; - } - // additions: - if !additions.is_empty() { - obkv.insert( - DelAdd::Addition, - CboRoaringBitmapCodec::bytes_encode(additions).map_err(|_| { - SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) } - })?, - )?; - } - - // insert everything in the same writer. - writer.insert(word.as_bytes(), obkv.into_inner().unwrap())?; - - Ok(()) -} diff --git a/crates/milli/src/update/new/extract/documents.rs b/crates/milli/src/update/new/extract/documents.rs index 31d2ada0f..5f287851a 100644 --- a/crates/milli/src/update/new/extract/documents.rs +++ b/crates/milli/src/update/new/extract/documents.rs @@ -240,12 +240,12 @@ impl<'extractor> SettingsChangeExtractor<'extractor> for SettingsChangeDocumentE /// modifies them by adding or removing vector fields based on embedder actions, /// and then updates the database. #[tracing::instrument(level = "trace", skip_all, target = "indexing::documents::extract")] -pub fn update_database_documents<'indexer, 'extractor, MSP, SD>( +pub fn update_database_documents<'indexer, MSP, SD>( documents: &'indexer DocumentsIndentifiers<'indexer>, indexing_context: IndexingContext, extractor_sender: &ExtractorBbqueueSender, settings_delta: &SD, - extractor_allocs: &'extractor mut ThreadLocal>, + extractor_allocs: &mut ThreadLocal>, ) -> Result<()> where MSP: Fn() -> bool + Sync, diff --git a/crates/milli/src/update/new/merger.rs b/crates/milli/src/update/new/merger.rs index 15f06c67d..44ba8e301 100644 --- a/crates/milli/src/update/new/merger.rs +++ b/crates/milli/src/update/new/merger.rs @@ -63,8 +63,8 @@ where } #[tracing::instrument(level = "trace", skip_all, target = "indexing::merge")] -pub fn merge_and_send_docids<'extractor, MSP, D>( - mut caches: Vec>, +pub fn merge_and_send_docids( + mut caches: Vec>, database: Database, index: &Index, docids_sender: WordDocidsSender, @@ -91,8 +91,8 @@ where } #[tracing::instrument(level = "trace", skip_all, target = "indexing::merge")] -pub fn merge_and_send_facet_docids<'extractor>( - mut caches: Vec>, +pub fn merge_and_send_facet_docids( + mut caches: Vec>, database: FacetDatabases, index: &Index, rtxn: &RoTxn, diff --git a/crates/tracing-trace/src/main.rs b/crates/tracing-trace/src/main.rs index 4a3d26923..22c96ec78 100644 --- a/crates/tracing-trace/src/main.rs +++ b/crates/tracing-trace/src/main.rs @@ -59,7 +59,7 @@ fn fibo_recursive(n: u32) -> u32 { if n == 1 { return 2; } - return fibo_recursive(n - 1) - fibo_recursive(n - 2); + fibo_recursive(n - 1) - fibo_recursive(n - 2) } use tracing_error::ExtractSpanTrace as _; From 69a84fbfe605c22bb905f1010afbba77f23cd519 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 18 Aug 2025 16:15:20 +0200 Subject: [PATCH 30/62] update to v1.22 --- Cargo.lock | 34 +++++++++---------- Cargo.toml | 2 +- .../after_processing_everything.snap | 4 +-- .../register_automatic_upgrade_task.snap | 2 +- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 4 +-- .../upgrade_task_failed_again.snap | 4 +-- .../upgrade_task_succeeded.snap | 4 +-- crates/index-scheduler/src/upgrade/mod.rs | 1 + crates/meilisearch/tests/upgrade/mod.rs | 4 +-- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- crates/milli/src/update/upgrade/mod.rs | 16 ++++----- .../upgrade/{v1_18.rs => new_hannoy.rs} | 6 ++-- 20 files changed, 50 insertions(+), 49 deletions(-) rename crates/milli/src/update/upgrade/{v1_18.rs => new_hannoy.rs} (90%) diff --git a/Cargo.lock b/Cargo.lock index 17f3b3daf..9ef2372f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "bumpalo", @@ -790,7 +790,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "time", @@ -1784,7 +1784,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "big_s", @@ -2016,7 +2016,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.19.0" +version = "1.22.0" dependencies = [ "tempfile", "thiserror 2.0.14", @@ -2038,7 +2038,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.19.0" +version = "1.22.0" dependencies = [ "insta", "levenshtein_automata", @@ -2060,7 +2060,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.19.0" +version = "1.22.0" dependencies = [ "criterion", "serde_json", @@ -2205,7 +2205,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.19.0" +version = "1.22.0" dependencies = [ "arbitrary", "bumpalo", @@ -3028,7 +3028,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "backoff", @@ -3275,7 +3275,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.19.0" +version = "1.22.0" dependencies = [ "criterion", "serde_json", @@ -3775,7 +3775,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.19.0" +version = "1.22.0" dependencies = [ "insta", "md5", @@ -3786,7 +3786,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.19.0" +version = "1.22.0" dependencies = [ "actix-cors", "actix-http", @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.19.0" +version = "1.22.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3902,7 +3902,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.19.0" +version = "1.22.0" dependencies = [ "actix-web", "anyhow", @@ -3937,7 +3937,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "clap", @@ -3971,7 +3971,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.19.0" +version = "1.22.0" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4554,7 +4554,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.19.0" +version = "1.22.0" dependencies = [ "big_s", "serde_json", @@ -7515,7 +7515,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.19.0" +version = "1.22.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index 836e08dcb..f4a6372a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.19.0" +version = "1.22.0" authors = [ "Quentin de Quelen ", "Clément Renault ", diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 90fb618f1..e3521f78d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index 9a3edc8ed..677bb9a07 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index fdaecb1b8..0833b0cb8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index fc28df258..eaec8bd70 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index d09780553..fb1163613 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index aa687425e..9dd2a5642 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index 4ca594936..c4b281d54 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -43,6 +43,7 @@ pub fn upgrade_index_scheduler( (1, 17, _) => 0, (1, 18, _) => 0, (1, 19, _) => 0, + (1, 22, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index e259b7c42..bf88352c1 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.19.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.22.0"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.19.1 is higher than the Meilisearch version 1.19.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.22.1 is higher than the Meilisearch version 1.22.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 8ca0ef6ac..c239bd89e 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 8ca0ef6ac..c239bd89e 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 8ca0ef6ac..c239bd89e 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index ffeabf4ca..465890fd0 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index ffeabf4ca..465890fd0 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index ffeabf4ca..465890fd0 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 5267a2edd..c6671f9c2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 1b482a69e..8eac19d6c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.19.0" + "upgradeTo": "v1.22.0" }, "error": null, "duration": "[duration]", diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 8f6a74b00..aad339378 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -1,18 +1,18 @@ +mod new_hannoy; mod v1_12; mod v1_13; mod v1_14; mod v1_15; mod v1_16; mod v1_17; -mod v1_18; use heed::RwTxn; +use new_hannoy::Latest_V1_18_New_Hannoy; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_15::Latest_V1_14_To_Latest_V1_15; use v1_16::Latest_V1_15_To_V1_16_0; -use v1_18::Latest_V1_17_To_V1_18_0; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; @@ -38,8 +38,8 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, - &ToTargetNoOp { target: (1, 17, 0) }, - &Latest_V1_17_To_V1_18_0 {}, + &ToTargetNoOp { target: (1, 18, 0) }, + &Latest_V1_18_New_Hannoy {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -67,10 +67,10 @@ const fn start(from: (u32, u32, u32)) -> Option { (1, 14, _) => function_index!(5), // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), - (1, 16, _) => function_index!(7), - (1, 17, _) => function_index!(8), - (1, 18, _) => function_index!(9), - (1, 19, _) => function_index!(9), + (1, 16, _) | (1, 17, _) => function_index!(7), + (1, 18, _) => function_index!(8), + (1, 19, _) => function_index!(8), + (1, 22, _) => function_index!(9), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/v1_18.rs b/crates/milli/src/update/upgrade/new_hannoy.rs similarity index 90% rename from crates/milli/src/update/upgrade/v1_18.rs rename to crates/milli/src/update/upgrade/new_hannoy.rs index f2e44f0f3..f4e446335 100644 --- a/crates/milli/src/update/upgrade/v1_18.rs +++ b/crates/milli/src/update/upgrade/new_hannoy.rs @@ -6,9 +6,9 @@ use crate::vector::VectorStore; use crate::{Index, Result}; #[allow(non_camel_case_types)] -pub(super) struct Latest_V1_17_To_V1_18_0(); +pub(super) struct Latest_V1_18_New_Hannoy(); -impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { +impl UpgradeIndex for Latest_V1_18_New_Hannoy { fn upgrade( &self, wtxn: &mut RwTxn, @@ -31,6 +31,6 @@ impl UpgradeIndex for Latest_V1_17_To_V1_18_0 { } fn target_version(&self) -> (u32, u32, u32) { - (1, 18, 0) + (1, 22, 0) } } From fb7ccc0db3ebed2dfedab9ba162c514d97f5abd1 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 18 Aug 2025 16:48:37 +0200 Subject: [PATCH 31/62] remove v1_17 --- crates/milli/src/update/upgrade/mod.rs | 1 - crates/milli/src/update/upgrade/v1_17.rs | 24 ------------------------ 2 files changed, 25 deletions(-) delete mode 100644 crates/milli/src/update/upgrade/v1_17.rs diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index aad339378..b7ff45b77 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -4,7 +4,6 @@ mod v1_13; mod v1_14; mod v1_15; mod v1_16; -mod v1_17; use heed::RwTxn; use new_hannoy::Latest_V1_18_New_Hannoy; diff --git a/crates/milli/src/update/upgrade/v1_17.rs b/crates/milli/src/update/upgrade/v1_17.rs deleted file mode 100644 index 412aa21d8..000000000 --- a/crates/milli/src/update/upgrade/v1_17.rs +++ /dev/null @@ -1,24 +0,0 @@ -use heed::RwTxn; - -use super::UpgradeIndex; -use crate::progress::Progress; -use crate::{Index, Result}; - -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_16_To_V1_17_0(); - -impl UpgradeIndex for Latest_V1_16_To_V1_17_0 { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - (1, 17, 0) - } -} From b5f0c194064836340157ff635b59271daa71e915 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 26 Aug 2025 16:32:17 +0200 Subject: [PATCH 32/62] Split the `vector` module in submodules --- crates/dump/src/reader/v6/mod.rs | 2 +- crates/milli/src/error.rs | 2 +- .../src/update/new/extract/vectors/mod.rs | 2 +- crates/milli/src/update/settings.rs | 50 +- crates/milli/src/vector/distribution.rs | 128 ++ .../src/vector/{ => embedder}/composite.rs | 12 +- crates/milli/src/vector/{ => embedder}/hf.rs | 5 +- .../milli/src/vector/{ => embedder}/manual.rs | 5 +- crates/milli/src/vector/embedder/mod.rs | 381 +++++ .../milli/src/vector/{ => embedder}/ollama.rs | 8 +- .../milli/src/vector/{ => embedder}/openai.rs | 7 +- .../milli/src/vector/{ => embedder}/rest.rs | 12 +- crates/milli/src/vector/embeddings.rs | 76 + crates/milli/src/vector/error.rs | 4 +- crates/milli/src/vector/mod.rs | 1446 +---------------- crates/milli/src/vector/runtime.rs | 81 + crates/milli/src/vector/session.rs | 3 +- crates/milli/src/vector/settings.rs | 121 +- crates/milli/src/vector/store.rs | 775 +++++++++ 19 files changed, 1557 insertions(+), 1563 deletions(-) create mode 100644 crates/milli/src/vector/distribution.rs rename crates/milli/src/vector/{ => embedder}/composite.rs (97%) rename crates/milli/src/vector/{ => embedder}/hf.rs (98%) rename crates/milli/src/vector/{ => embedder}/manual.rs (93%) create mode 100644 crates/milli/src/vector/embedder/mod.rs rename crates/milli/src/vector/{ => embedder}/ollama.rs (96%) rename crates/milli/src/vector/{ => embedder}/openai.rs (98%) rename crates/milli/src/vector/{ => embedder}/rest.rs (98%) create mode 100644 crates/milli/src/vector/embeddings.rs create mode 100644 crates/milli/src/vector/runtime.rs create mode 100644 crates/milli/src/vector/store.rs diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 75ff2ebe6..b5549ec65 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -4,7 +4,7 @@ use std::io::{BufRead, BufReader, ErrorKind}; use std::path::Path; pub use meilisearch_types::milli; -use meilisearch_types::milli::vector::hf::OverridePooling; +use meilisearch_types::milli::vector::embedder::hf::OverridePooling; use tempfile::TempDir; use time::OffsetDateTime; use tracing::debug; diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 787f42753..11d7756c1 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -355,7 +355,7 @@ and can not be more than 511 bytes.", .document_id.to_string() context: crate::vector::settings::NestingContext, field: crate::vector::settings::MetaEmbeddingSetting, }, - #[error("`.embedders.{embedder_name}.model`: Invalid model `{model}` for OpenAI. Supported models: {:?}", crate::vector::openai::EmbeddingModel::supported_models())] + #[error("`.embedders.{embedder_name}.model`: Invalid model `{model}` for OpenAI. Supported models: {:?}", crate::vector::embedder::openai::EmbeddingModel::supported_models())] InvalidOpenAiModel { embedder_name: String, model: String }, #[error("`.embedders.{embedder_name}`: Missing field `{field}` (note: this field is mandatory for source `{source_}`)")] MissingFieldForSource { diff --git a/crates/milli/src/update/new/extract/vectors/mod.rs b/crates/milli/src/update/new/extract/vectors/mod.rs index 71fa9bf09..f147de360 100644 --- a/crates/milli/src/update/new/extract/vectors/mod.rs +++ b/crates/milli/src/update/new/extract/vectors/mod.rs @@ -475,7 +475,7 @@ impl<'doc> OnEmbed<'doc> for OnEmbeddingDocumentUpdates<'doc, '_> { } fn process_embedding_error( &mut self, - error: crate::vector::hf::EmbedError, + error: crate::vector::error::EmbedError, embedder_name: &'doc str, unused_vectors_distribution: &UnusedVectorsDistributionBump, metadata: BVec<'doc, Metadata<'doc>>, diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index bca8fbc59..5530ae718 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -33,6 +33,7 @@ use crate::update::index_documents::IndexDocumentsMethod; use crate::update::new::indexer::reindex; use crate::update::{IndexDocuments, UpdateIndexingStep}; use crate::vector::db::{FragmentConfigs, IndexEmbeddingConfig}; +use crate::vector::embedder::{openai, rest}; use crate::vector::json_template::JsonTemplate; use crate::vector::settings::{ EmbedderAction, EmbedderSource, EmbeddingSettings, EmbeddingValidationContext, NestingContext, @@ -2208,39 +2209,29 @@ pub fn validate_embedding_settings( if let Some(request) = request.as_ref().set() { let request = match with_fragments { WithFragments::Yes { indexing_fragments, search_fragments } => { - crate::vector::rest::RequestData::new( - request.to_owned(), - indexing_fragments, - search_fragments, - ) - .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())) + rest::RequestData::new(request.to_owned(), indexing_fragments, search_fragments) + .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())) + } + WithFragments::No => { + rest::RequestData::new(request.to_owned(), Default::default(), Default::default()) + .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())) } - WithFragments::No => crate::vector::rest::RequestData::new( - request.to_owned(), - Default::default(), - Default::default(), - ) - .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())), WithFragments::Maybe => { let mut indexing_fragments = BTreeMap::new(); indexing_fragments.insert("test".to_string(), serde_json::json!("test")); - crate::vector::rest::RequestData::new( - request.to_owned(), - indexing_fragments, - Default::default(), - ) - .or_else(|_| { - crate::vector::rest::RequestData::new( - request.to_owned(), - Default::default(), - Default::default(), - ) - }) - .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())) + rest::RequestData::new(request.to_owned(), indexing_fragments, Default::default()) + .or_else(|_| { + rest::RequestData::new( + request.to_owned(), + Default::default(), + Default::default(), + ) + }) + .map_err(|error| crate::UserError::VectorEmbeddingError(error.into())) } }?; if let Some(response) = response.as_ref().set() { - crate::vector::rest::Response::new(response.to_owned(), &request) + rest::Response::new(response.to_owned(), &request) .map_err(|error| crate::UserError::VectorEmbeddingError(error.into()))?; } } @@ -2293,11 +2284,12 @@ pub fn validate_embedding_settings( match inferred_source { EmbedderSource::OpenAi => { if let Setting::Set(model) = &model { - let model = crate::vector::openai::EmbeddingModel::from_name(model.as_str()) - .ok_or(crate::error::UserError::InvalidOpenAiModel { + let model = openai::EmbeddingModel::from_name(model.as_str()).ok_or( + crate::error::UserError::InvalidOpenAiModel { embedder_name: name.to_owned(), model: model.clone(), - })?; + }, + )?; if let Setting::Set(dimensions) = dimensions { if !model.supports_overriding_dimensions() && dimensions != model.default_dimensions() diff --git a/crates/milli/src/vector/distribution.rs b/crates/milli/src/vector/distribution.rs new file mode 100644 index 000000000..b17ad9204 --- /dev/null +++ b/crates/milli/src/vector/distribution.rs @@ -0,0 +1,128 @@ +use deserr::{DeserializeError, Deserr}; +use ordered_float::OrderedFloat; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Describes the mean and sigma of distribution of embedding similarity in the embedding space. +/// +/// The intended use is to make the similarity score more comparable to the regular ranking score. +/// This allows to correct effects where results are too "packed" around a certain value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize, ToSchema)] +#[serde(from = "DistributionShiftSerializable")] +#[serde(into = "DistributionShiftSerializable")] +pub struct DistributionShift { + /// Value where the results are "packed". + /// + /// Similarity scores are translated so that they are packed around 0.5 instead + #[schema(value_type = f32)] + pub current_mean: OrderedFloat, + + /// standard deviation of a similarity score. + /// + /// Set below 0.4 to make the results less packed around the mean, and above 0.4 to make them more packed. + #[schema(value_type = f32)] + pub current_sigma: OrderedFloat, +} + +impl Deserr for DistributionShift +where + E: DeserializeError, +{ + fn deserialize_from_value( + value: deserr::Value, + location: deserr::ValuePointerRef<'_>, + ) -> Result { + let value = DistributionShiftSerializable::deserialize_from_value(value, location)?; + if value.mean < 0. || value.mean > 1. { + return Err(deserr::take_cf_content(E::error::( + None, + deserr::ErrorKind::Unexpected { + msg: format!( + "the distribution mean must be in the range [0, 1], got {}", + value.mean + ), + }, + location, + ))); + } + if value.sigma <= 0. || value.sigma > 1. { + return Err(deserr::take_cf_content(E::error::( + None, + deserr::ErrorKind::Unexpected { + msg: format!( + "the distribution sigma must be in the range ]0, 1], got {}", + value.sigma + ), + }, + location, + ))); + } + + Ok(value.into()) + } +} + +#[derive(Serialize, Deserialize, Deserr)] +#[serde(deny_unknown_fields)] +#[deserr(deny_unknown_fields)] +struct DistributionShiftSerializable { + mean: f32, + sigma: f32, +} + +impl From for DistributionShiftSerializable { + fn from( + DistributionShift { + current_mean: OrderedFloat(current_mean), + current_sigma: OrderedFloat(current_sigma), + }: DistributionShift, + ) -> Self { + Self { mean: current_mean, sigma: current_sigma } + } +} + +impl From for DistributionShift { + fn from(DistributionShiftSerializable { mean, sigma }: DistributionShiftSerializable) -> Self { + Self { current_mean: OrderedFloat(mean), current_sigma: OrderedFloat(sigma) } + } +} + +impl DistributionShift { + /// `None` if sigma <= 0. + pub fn new(mean: f32, sigma: f32) -> Option { + if sigma <= 0.0 { + None + } else { + Some(Self { current_mean: OrderedFloat(mean), current_sigma: OrderedFloat(sigma) }) + } + } + + pub fn shift(&self, score: f32) -> f32 { + let current_mean = self.current_mean.0; + let current_sigma = self.current_sigma.0; + // + // We're somewhat abusively mapping the distribution of distances to a gaussian. + // The parameters we're given is the mean and sigma of the native result distribution. + // We're using them to retarget the distribution to a gaussian centered on 0.5 with a sigma of 0.4. + + let target_mean = 0.5; + let target_sigma = 0.4; + + // a^2 sig1^2 = sig2^2 => a^2 = sig2^2 / sig1^2 => a = sig2 / sig1, assuming a, sig1, and sig2 positive. + let factor = target_sigma / current_sigma; + // a*mu1 + b = mu2 => b = mu2 - a*mu1 + let offset = target_mean - (factor * current_mean); + + let mut score = factor * score + offset; + + // clamp the final score in the ]0, 1] interval. + if score <= 0.0 { + score = f32::EPSILON; + } + if score > 1.0 { + score = 1.0; + } + + score + } +} diff --git a/crates/milli/src/vector/composite.rs b/crates/milli/src/vector/embedder/composite.rs similarity index 97% rename from crates/milli/src/vector/composite.rs rename to crates/milli/src/vector/embedder/composite.rs index 539e92ba8..c34c31b41 100644 --- a/crates/milli/src/vector/composite.rs +++ b/crates/milli/src/vector/embedder/composite.rs @@ -2,14 +2,14 @@ use std::time::Instant; use hannoy::Distance; -use super::error::CompositeEmbedderContainsHuggingFace; -use super::{ - hf, manual, ollama, openai, rest, DistributionShift, EmbedError, Embedding, EmbeddingCache, - NewEmbedderError, -}; +use super::{hf, manual, ollama, openai, rest, Embedding, EmbeddingCache}; use crate::progress::EmbedderStats; +use crate::vector::error::{CompositeEmbedderContainsHuggingFace, EmbedError, NewEmbedderError}; +use crate::vector::DistributionShift; use crate::ThreadPoolNoAbort; +pub(in crate::vector) const MAX_COMPOSITE_DISTANCE: f32 = 0.01; + #[derive(Debug)] pub enum SubEmbedder { /// An embedder based on running local models, fetched from the Hugging Face Hub. @@ -336,7 +336,7 @@ fn check_similarity( }; let distance = hannoy::distances::Cosine::distance(&left, &right); - if distance > super::MAX_COMPOSITE_DISTANCE { + if distance > crate::vector::embedder::composite::MAX_COMPOSITE_DISTANCE { return Err(NewEmbedderError::composite_embedding_value_mismatch(distance, hint)); } } diff --git a/crates/milli/src/vector/hf.rs b/crates/milli/src/vector/embedder/hf.rs similarity index 98% rename from crates/milli/src/vector/hf.rs rename to crates/milli/src/vector/embedder/hf.rs index 1e5c7bd1c..18f80dec1 100644 --- a/crates/milli/src/vector/hf.rs +++ b/crates/milli/src/vector/embedder/hf.rs @@ -6,8 +6,9 @@ use hf_hub::api::sync::Api; use hf_hub::{Repo, RepoType}; use tokenizers::{PaddingParams, Tokenizer}; -pub use super::error::{EmbedError, Error, NewEmbedderError}; -use super::{DistributionShift, Embedding, EmbeddingCache}; +use super::EmbeddingCache; +use crate::vector::error::{EmbedError, NewEmbedderError}; +use crate::vector::{DistributionShift, Embedding}; #[derive( Debug, diff --git a/crates/milli/src/vector/manual.rs b/crates/milli/src/vector/embedder/manual.rs similarity index 93% rename from crates/milli/src/vector/manual.rs rename to crates/milli/src/vector/embedder/manual.rs index b95bf0ea2..132aab0bf 100644 --- a/crates/milli/src/vector/manual.rs +++ b/crates/milli/src/vector/embedder/manual.rs @@ -1,6 +1,5 @@ -use super::error::EmbedError; -use super::DistributionShift; -use crate::vector::Embedding; +use crate::vector::error::EmbedError; +use crate::vector::{DistributionShift, Embedding}; #[derive(Debug, Clone, Copy)] pub struct Embedder { diff --git a/crates/milli/src/vector/embedder/mod.rs b/crates/milli/src/vector/embedder/mod.rs new file mode 100644 index 000000000..b7f7b1de4 --- /dev/null +++ b/crates/milli/src/vector/embedder/mod.rs @@ -0,0 +1,381 @@ +pub mod composite; +pub mod hf; +pub mod manual; +pub mod ollama; +pub mod openai; +pub mod rest; + +use std::num::NonZeroUsize; +use std::sync::Mutex; +use std::time::Instant; + +use composite::SubEmbedderOptions; + +use crate::progress::EmbedderStats; +use crate::prompt::PromptData; +use crate::vector::error::{EmbedError, NewEmbedderError}; +use crate::vector::{DistributionShift, Embedding}; +use crate::ThreadPoolNoAbort; + +/// An embedder can be used to transform text into embeddings. +#[derive(Debug)] +pub enum Embedder { + /// An embedder based on running local models, fetched from the Hugging Face Hub. + HuggingFace(hf::Embedder), + /// An embedder based on making embedding queries against the OpenAI API. + OpenAi(openai::Embedder), + /// An embedder based on the user providing the embeddings in the documents and queries. + UserProvided(manual::Embedder), + /// An embedder based on making embedding queries against an embedding server. + Ollama(ollama::Embedder), + /// An embedder based on making embedding queries against a generic JSON/REST embedding server. + Rest(rest::Embedder), + /// An embedder composed of an embedder at search time and an embedder at indexing time. + Composite(composite::Embedder), +} + +/// Configuration for an embedder. +#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)] +pub struct EmbeddingConfig { + /// Options of the embedder, specific to each kind of embedder + pub embedder_options: EmbedderOptions, + /// Document template + pub prompt: PromptData, + /// If this embedder is binary quantized + pub quantized: Option, + // TODO: add metrics and anything needed +} + +impl EmbeddingConfig { + pub fn quantized(&self) -> bool { + self.quantized.unwrap_or_default() + } +} + +/// Options of an embedder, specific to each kind of embedder. +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +pub enum EmbedderOptions { + HuggingFace(hf::EmbedderOptions), + OpenAi(openai::EmbedderOptions), + Ollama(ollama::EmbedderOptions), + UserProvided(manual::EmbedderOptions), + Rest(rest::EmbedderOptions), + Composite(composite::EmbedderOptions), +} + +impl EmbedderOptions { + pub fn fragment(&self, name: &str) -> Option<&serde_json::Value> { + match &self { + EmbedderOptions::HuggingFace(_) + | EmbedderOptions::OpenAi(_) + | EmbedderOptions::Ollama(_) + | EmbedderOptions::UserProvided(_) => None, + EmbedderOptions::Rest(embedder_options) => { + embedder_options.indexing_fragments.get(name) + } + EmbedderOptions::Composite(embedder_options) => { + if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index { + embedder_options.indexing_fragments.get(name) + } else { + None + } + } + } + } + + pub fn has_fragments(&self) -> bool { + match &self { + EmbedderOptions::HuggingFace(_) + | EmbedderOptions::OpenAi(_) + | EmbedderOptions::Ollama(_) + | EmbedderOptions::UserProvided(_) => false, + EmbedderOptions::Rest(embedder_options) => { + !embedder_options.indexing_fragments.is_empty() + } + EmbedderOptions::Composite(embedder_options) => { + if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index { + !embedder_options.indexing_fragments.is_empty() + } else { + false + } + } + } + } +} + +impl Default for EmbedderOptions { + fn default() -> Self { + Self::HuggingFace(Default::default()) + } +} + +impl Embedder { + /// Spawns a new embedder built from its options. + pub fn new( + options: EmbedderOptions, + cache_cap: usize, + ) -> std::result::Result { + Ok(match options { + EmbedderOptions::HuggingFace(options) => { + Self::HuggingFace(hf::Embedder::new(options, cache_cap)?) + } + EmbedderOptions::OpenAi(options) => { + Self::OpenAi(openai::Embedder::new(options, cache_cap)?) + } + EmbedderOptions::Ollama(options) => { + Self::Ollama(ollama::Embedder::new(options, cache_cap)?) + } + EmbedderOptions::UserProvided(options) => { + Self::UserProvided(manual::Embedder::new(options)) + } + EmbedderOptions::Rest(options) => Self::Rest(rest::Embedder::new( + options, + cache_cap, + rest::ConfigurationSource::User, + )?), + EmbedderOptions::Composite(options) => { + Self::Composite(composite::Embedder::new(options, cache_cap)?) + } + }) + } + + /// Embed in search context + + #[tracing::instrument(level = "debug", skip_all, target = "search")] + pub fn embed_search( + &self, + query: SearchQuery<'_>, + deadline: Option, + ) -> std::result::Result { + match query { + SearchQuery::Text(text) => self.embed_search_text(text, deadline), + SearchQuery::Media { q, media } => self.embed_search_media(q, media, deadline), + } + } + + pub fn embed_search_text( + &self, + text: &str, + deadline: Option, + ) -> std::result::Result { + if let Some(cache) = self.cache() { + if let Some(embedding) = cache.get(text) { + tracing::trace!(text, "embedding found in cache"); + return Ok(embedding); + } + } + let embedding = match self { + Embedder::HuggingFace(embedder) => embedder.embed_one(text), + Embedder::OpenAi(embedder) => embedder + .embed(&[text], deadline, None)? + .pop() + .ok_or_else(EmbedError::missing_embedding), + Embedder::Ollama(embedder) => embedder + .embed(&[text], deadline, None)? + .pop() + .ok_or_else(EmbedError::missing_embedding), + Embedder::UserProvided(embedder) => embedder.embed_one(text), + Embedder::Rest(embedder) => embedder.embed_one(SearchQuery::Text(text), deadline, None), + Embedder::Composite(embedder) => embedder.search.embed_one(text, deadline, None), + }?; + + if let Some(cache) = self.cache() { + cache.put(text.to_owned(), embedding.clone()); + } + + Ok(embedding) + } + + pub fn embed_search_media( + &self, + q: Option<&str>, + media: Option<&serde_json::Value>, + deadline: Option, + ) -> std::result::Result { + let Embedder::Rest(embedder) = self else { + return Err(EmbedError::rest_media_not_a_rest()); + }; + embedder.embed_one(SearchQuery::Media { q, media }, deadline, None) + } + + /// Embed multiple chunks of texts. + /// + /// Each chunk is composed of one or multiple texts. + pub fn embed_index( + &self, + text_chunks: Vec>, + threads: &ThreadPoolNoAbort, + embedder_stats: &EmbedderStats, + ) -> std::result::Result>, EmbedError> { + match self { + Embedder::HuggingFace(embedder) => embedder.embed_index(text_chunks), + Embedder::OpenAi(embedder) => { + embedder.embed_index(text_chunks, threads, embedder_stats) + } + Embedder::Ollama(embedder) => { + embedder.embed_index(text_chunks, threads, embedder_stats) + } + Embedder::UserProvided(embedder) => embedder.embed_index(text_chunks), + Embedder::Rest(embedder) => embedder.embed_index(text_chunks, threads, embedder_stats), + Embedder::Composite(embedder) => { + embedder.index.embed_index(text_chunks, threads, embedder_stats) + } + } + } + + /// Non-owning variant of [`Self::embed_index`]. + pub fn embed_index_ref( + &self, + texts: &[&str], + threads: &ThreadPoolNoAbort, + embedder_stats: &EmbedderStats, + ) -> std::result::Result, EmbedError> { + match self { + Embedder::HuggingFace(embedder) => embedder.embed_index_ref(texts), + Embedder::OpenAi(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), + Embedder::Ollama(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), + Embedder::UserProvided(embedder) => embedder.embed_index_ref(texts), + Embedder::Rest(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), + Embedder::Composite(embedder) => { + embedder.index.embed_index_ref(texts, threads, embedder_stats) + } + } + } + + pub fn embed_index_ref_fragments( + &self, + fragments: &[serde_json::Value], + threads: &ThreadPoolNoAbort, + embedder_stats: &EmbedderStats, + ) -> std::result::Result, EmbedError> { + if let Embedder::Rest(embedder) = self { + embedder.embed_index_ref(fragments, threads, embedder_stats) + } else { + let Embedder::Composite(embedder) = self else { + unimplemented!("embedding fragments is only available for rest embedders") + }; + let crate::vector::embedder::composite::SubEmbedder::Rest(embedder) = &embedder.index + else { + unimplemented!("embedding fragments is only available for rest embedders") + }; + + embedder.embed_index_ref(fragments, threads, embedder_stats) + } + } + + /// Indicates the preferred number of chunks to pass to [`Self::embed_chunks`] + pub fn chunk_count_hint(&self) -> usize { + match self { + Embedder::HuggingFace(embedder) => embedder.chunk_count_hint(), + Embedder::OpenAi(embedder) => embedder.chunk_count_hint(), + Embedder::Ollama(embedder) => embedder.chunk_count_hint(), + Embedder::UserProvided(_) => 100, + Embedder::Rest(embedder) => embedder.chunk_count_hint(), + Embedder::Composite(embedder) => embedder.index.chunk_count_hint(), + } + } + + /// Indicates the preferred number of texts in a single chunk passed to [`Self::embed`] + pub fn prompt_count_in_chunk_hint(&self) -> usize { + match self { + Embedder::HuggingFace(embedder) => embedder.prompt_count_in_chunk_hint(), + Embedder::OpenAi(embedder) => embedder.prompt_count_in_chunk_hint(), + Embedder::Ollama(embedder) => embedder.prompt_count_in_chunk_hint(), + Embedder::UserProvided(_) => 1, + Embedder::Rest(embedder) => embedder.prompt_count_in_chunk_hint(), + Embedder::Composite(embedder) => embedder.index.prompt_count_in_chunk_hint(), + } + } + + /// Indicates the dimensions of a single embedding produced by the embedder. + pub fn dimensions(&self) -> usize { + match self { + Embedder::HuggingFace(embedder) => embedder.dimensions(), + Embedder::OpenAi(embedder) => embedder.dimensions(), + Embedder::Ollama(embedder) => embedder.dimensions(), + Embedder::UserProvided(embedder) => embedder.dimensions(), + Embedder::Rest(embedder) => embedder.dimensions(), + Embedder::Composite(embedder) => embedder.dimensions(), + } + } + + /// An optional distribution used to apply an affine transformation to the similarity score of a document. + pub fn distribution(&self) -> Option { + match self { + Embedder::HuggingFace(embedder) => embedder.distribution(), + Embedder::OpenAi(embedder) => embedder.distribution(), + Embedder::Ollama(embedder) => embedder.distribution(), + Embedder::UserProvided(embedder) => embedder.distribution(), + Embedder::Rest(embedder) => embedder.distribution(), + Embedder::Composite(embedder) => embedder.distribution(), + } + } + + pub fn uses_document_template(&self) -> bool { + match self { + Embedder::HuggingFace(_) + | Embedder::OpenAi(_) + | Embedder::Ollama(_) + | Embedder::Rest(_) => true, + Embedder::UserProvided(_) => false, + Embedder::Composite(embedder) => embedder.index.uses_document_template(), + } + } + + fn cache(&self) -> Option<&EmbeddingCache> { + match self { + Embedder::HuggingFace(embedder) => Some(embedder.cache()), + Embedder::OpenAi(embedder) => Some(embedder.cache()), + Embedder::UserProvided(_) => None, + Embedder::Ollama(embedder) => Some(embedder.cache()), + Embedder::Rest(embedder) => Some(embedder.cache()), + Embedder::Composite(embedder) => embedder.search.cache(), + } + } +} + +#[derive(Clone, Copy)] +pub enum SearchQuery<'a> { + Text(&'a str), + Media { q: Option<&'a str>, media: Option<&'a serde_json::Value> }, +} + +#[derive(Debug)] +struct EmbeddingCache { + data: Option>>, +} + +impl EmbeddingCache { + const MAX_TEXT_LEN: usize = 2000; + + pub fn new(cap: usize) -> Self { + let data = NonZeroUsize::new(cap).map(lru::LruCache::new).map(Mutex::new); + Self { data } + } + + /// Get the embedding corresponding to `text`, if any is present in the cache. + pub fn get(&self, text: &str) -> Option { + let data = self.data.as_ref()?; + if text.len() > Self::MAX_TEXT_LEN { + return None; + } + let mut cache = data.lock().unwrap(); + + cache.get(text).cloned() + } + + /// Puts a new embedding for the specified `text` + pub fn put(&self, text: String, embedding: Embedding) { + let Some(data) = self.data.as_ref() else { + return; + }; + if text.len() > Self::MAX_TEXT_LEN { + return; + } + tracing::trace!(text, "embedding added to cache"); + + let mut cache = data.lock().unwrap(); + + cache.put(text, embedding); + } +} diff --git a/crates/milli/src/vector/ollama.rs b/crates/milli/src/vector/embedder/ollama.rs similarity index 96% rename from crates/milli/src/vector/ollama.rs rename to crates/milli/src/vector/embedder/ollama.rs index feec92cc0..6e2dc185f 100644 --- a/crates/milli/src/vector/ollama.rs +++ b/crates/milli/src/vector/embedder/ollama.rs @@ -3,12 +3,12 @@ use std::time::Instant; use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _}; use rayon::slice::ParallelSlice as _; -use super::error::{EmbedError, EmbedErrorKind, NewEmbedderError, NewEmbedderErrorKind}; use super::rest::{Embedder as RestEmbedder, EmbedderOptions as RestEmbedderOptions}; -use super::{DistributionShift, EmbeddingCache, REQUEST_PARALLELISM}; +use super::EmbeddingCache; use crate::error::FaultSource; use crate::progress::EmbedderStats; -use crate::vector::Embedding; +use crate::vector::error::{EmbedError, EmbedErrorKind, NewEmbedderError, NewEmbedderErrorKind}; +use crate::vector::{DistributionShift, Embedding, REQUEST_PARALLELISM}; use crate::ThreadPoolNoAbort; #[derive(Debug)] @@ -88,7 +88,7 @@ impl Embedder { Err(NewEmbedderError { kind: NewEmbedderErrorKind::CouldNotDetermineDimension(EmbedError { - kind: super::error::EmbedErrorKind::RestOtherStatusCode(404, error), + kind: EmbedErrorKind::RestOtherStatusCode(404, error), fault: _, }), fault: _, diff --git a/crates/milli/src/vector/openai.rs b/crates/milli/src/vector/embedder/openai.rs similarity index 98% rename from crates/milli/src/vector/openai.rs rename to crates/milli/src/vector/embedder/openai.rs index bf6c92978..4fec228e4 100644 --- a/crates/milli/src/vector/openai.rs +++ b/crates/milli/src/vector/embedder/openai.rs @@ -5,13 +5,12 @@ use ordered_float::OrderedFloat; use rayon::iter::{IntoParallelIterator, ParallelIterator as _}; use rayon::slice::ParallelSlice as _; -use super::error::{EmbedError, NewEmbedderError}; use super::rest::{Embedder as RestEmbedder, EmbedderOptions as RestEmbedderOptions}; -use super::{DistributionShift, EmbeddingCache, REQUEST_PARALLELISM}; +use super::{DistributionShift, EmbeddingCache}; use crate::error::FaultSource; use crate::progress::EmbedderStats; -use crate::vector::error::EmbedErrorKind; -use crate::vector::Embedding; +use crate::vector::error::{EmbedError, EmbedErrorKind, NewEmbedderError}; +use crate::vector::{Embedding, REQUEST_PARALLELISM}; use crate::ThreadPoolNoAbort; #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] diff --git a/crates/milli/src/vector/rest.rs b/crates/milli/src/vector/embedder/rest.rs similarity index 98% rename from crates/milli/src/vector/rest.rs rename to crates/milli/src/vector/embedder/rest.rs index 64e4e74c7..3e0c5989a 100644 --- a/crates/milli/src/vector/rest.rs +++ b/crates/milli/src/vector/embedder/rest.rs @@ -8,14 +8,12 @@ use rayon::slice::ParallelSlice as _; use serde::{Deserialize, Serialize}; use serde_json::Value; -use super::error::EmbedErrorKind; -use super::json_template::{InjectableValue, JsonTemplate}; -use super::{ - DistributionShift, EmbedError, Embedding, EmbeddingCache, NewEmbedderError, SearchQuery, - REQUEST_PARALLELISM, -}; +use super::EmbeddingCache; use crate::error::FaultSource; use crate::progress::EmbedderStats; +use crate::vector::error::{EmbedError, EmbedErrorKind, NewEmbedderError}; +use crate::vector::json_template::{InjectableValue, JsonTemplate}; +use crate::vector::{DistributionShift, Embedding, SearchQuery, REQUEST_PARALLELISM}; use crate::ThreadPoolNoAbort; // retrying in case of failure @@ -315,7 +313,7 @@ impl Embedder { } pub fn chunk_count_hint(&self) -> usize { - super::REQUEST_PARALLELISM + crate::vector::REQUEST_PARALLELISM } pub fn prompt_count_in_chunk_hint(&self) -> usize { diff --git a/crates/milli/src/vector/embeddings.rs b/crates/milli/src/vector/embeddings.rs new file mode 100644 index 000000000..467ebc81e --- /dev/null +++ b/crates/milli/src/vector/embeddings.rs @@ -0,0 +1,76 @@ +/// One or multiple embeddings stored consecutively in a flat vector. +#[derive(Debug, PartialEq)] +pub struct Embeddings { + data: Vec, + dimension: usize, +} + +impl Embeddings { + /// Declares an empty vector of embeddings of the specified dimensions. + pub fn new(dimension: usize) -> Self { + Self { data: Default::default(), dimension } + } + + /// Declares a vector of embeddings containing a single element. + /// + /// The dimension is inferred from the length of the passed embedding. + pub fn from_single_embedding(embedding: Vec) -> Self { + Self { dimension: embedding.len(), data: embedding } + } + + /// Declares a vector of embeddings from its components. + /// + /// `data.len()` must be a multiple of `dimension`, otherwise an error is returned. + pub fn from_inner(data: Vec, dimension: usize) -> Result> { + let mut this = Self::new(dimension); + this.append(data)?; + Ok(this) + } + + /// Returns the number of embeddings in this vector of embeddings. + pub fn embedding_count(&self) -> usize { + self.data.len() / self.dimension + } + + /// Dimension of a single embedding. + pub fn dimension(&self) -> usize { + self.dimension + } + + /// Deconstructs self into the inner flat vector. + pub fn into_inner(self) -> Vec { + self.data + } + + /// A reference to the inner flat vector. + pub fn as_inner(&self) -> &[F] { + &self.data + } + + /// Iterates over the embeddings contained in the flat vector. + pub fn iter(&self) -> impl Iterator + '_ { + self.data.as_slice().chunks_exact(self.dimension) + } + + /// Push an embedding at the end of the embeddings. + /// + /// If `embedding.len() != self.dimension`, then the push operation fails. + pub fn push(&mut self, mut embedding: Vec) -> Result<(), Vec> { + if embedding.len() != self.dimension { + return Err(embedding); + } + self.data.append(&mut embedding); + Ok(()) + } + + /// Append a flat vector of embeddings at the end of the embeddings. + /// + /// If `embeddings.len() % self.dimension != 0`, then the append operation fails. + pub fn append(&mut self, mut embeddings: Vec) -> Result<(), Vec> { + if embeddings.len() % self.dimension != 0 { + return Err(embeddings); + } + self.data.append(&mut embeddings); + Ok(()) + } +} diff --git a/crates/milli/src/vector/error.rs b/crates/milli/src/vector/error.rs index 0d737cbfc..b4b90b24b 100644 --- a/crates/milli/src/vector/error.rs +++ b/crates/milli/src/vector/error.rs @@ -6,10 +6,10 @@ use hf_hub::api::sync::ApiError; use itertools::Itertools as _; use super::parsed_vectors::ParsedVectorsDiff; -use super::rest::ConfigurationSource; -use super::MAX_COMPOSITE_DISTANCE; use crate::error::FaultSource; use crate::update::new::vector_document::VectorDocument; +use crate::vector::embedder::composite::MAX_COMPOSITE_DISTANCE; +use crate::vector::embedder::rest::ConfigurationSource; use crate::{FieldDistribution, PanicCatched}; #[derive(Debug, thiserror::Error)] diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 9487cd9b1..f9b62d7d8 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -1,1453 +1,29 @@ -use std::collections::HashMap; -use std::num::NonZeroUsize; -use std::sync::{Arc, Mutex}; -use std::time::Instant; - -use deserr::{DeserializeError, Deserr}; -use hannoy::distances::{Cosine, Hamming}; -use hannoy::ItemId; -use heed::{RoTxn, RwTxn, Unspecified}; -use ordered_float::OrderedFloat; -use rand::SeedableRng as _; -use roaring::RoaringBitmap; -use serde::{Deserialize, Serialize}; -use utoipa::ToSchema; - -use self::error::{EmbedError, NewEmbedderError}; -use crate::progress::{EmbedderStats, Progress}; -use crate::prompt::{Prompt, PromptData}; -use crate::vector::composite::SubEmbedderOptions; -use crate::vector::json_template::JsonTemplate; -use crate::ThreadPoolNoAbort; - -pub mod composite; pub mod db; +mod distribution; +pub mod embedder; +mod embeddings; pub mod error; pub mod extractor; -pub mod hf; pub mod json_template; -pub mod manual; -pub mod openai; pub mod parsed_vectors; +mod runtime; pub mod session; pub mod settings; - -pub mod ollama; -pub mod rest; +mod store; pub use self::error::Error; pub type Embedding = Vec; +pub use distribution::DistributionShift; +pub use embedder::{Embedder, EmbedderOptions, EmbeddingConfig, SearchQuery}; +pub use embeddings::Embeddings; +pub use runtime::{RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment}; +pub use store::{HannoyStats, VectorStore}; + pub const REQUEST_PARALLELISM: usize = 40; -pub const MAX_COMPOSITE_DISTANCE: f32 = 0.01; - -const HANNOY_EF_CONSTRUCTION: usize = 125; -const HANNOY_M: usize = 16; -const HANNOY_M0: usize = 32; - -pub struct VectorStore { - version: (u32, u32, u32), - database: hannoy::Database, - embedder_index: u8, - quantized: bool, -} - -impl VectorStore { - pub fn new( - version: (u32, u32, u32), - database: hannoy::Database, - embedder_index: u8, - quantized: bool, - ) -> Self { - Self { version, database, embedder_index, quantized } - } - - pub fn embedder_index(&self) -> u8 { - self.embedder_index - } - - /// Whether we must use the arroy to read the vector store. - pub fn version_uses_arroy(&self) -> bool { - let (major, minor, _patch) = self.version; - major == 1 && minor < 18 - } - - fn arroy_readers<'a, D: arroy::Distance>( - &'a self, - rtxn: &'a RoTxn<'a>, - db: arroy::Database, - ) -> impl Iterator, arroy::Error>> + 'a { - vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { - match arroy::Reader::open(rtxn, index, db) { - Ok(reader) => match reader.is_empty(rtxn) { - Ok(false) => Some(Ok(reader)), - Ok(true) => None, - Err(e) => Some(Err(e)), - }, - Err(arroy::Error::MissingMetadata(_)) => None, - Err(e) => Some(Err(e)), - } - }) - } - - fn readers<'a, D: hannoy::Distance>( - &'a self, - rtxn: &'a RoTxn<'a>, - db: hannoy::Database, - ) -> impl Iterator, hannoy::Error>> + 'a { - vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { - match hannoy::Reader::open(rtxn, index, db) { - Ok(reader) => match reader.is_empty(rtxn) { - Ok(false) => Some(Ok(reader)), - Ok(true) => None, - Err(e) => Some(Err(e)), - }, - Err(hannoy::Error::MissingMetadata(_)) => None, - Err(e) => Some(Err(e)), - } - }) - } - - /// The item ids that are present in the store specified by its id. - /// - /// The ids are accessed via a lambda to avoid lifetime shenanigans. - pub fn items_in_store( - &self, - rtxn: &RoTxn, - store_id: u8, - with_items: F, - ) -> crate::Result - where - F: FnOnce(&RoaringBitmap) -> O, - { - if self.version_uses_arroy() { - if self.quantized { - self._arroy_items_in_store(rtxn, self.arroy_quantized_db(), store_id, with_items) - .map_err(Into::into) - } else { - self._arroy_items_in_store(rtxn, self.arroy_angular_db(), store_id, with_items) - .map_err(Into::into) - } - } else if self.quantized { - self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) - .map_err(Into::into) - } else { - self._items_in_store(rtxn, self.angular_db(), store_id, with_items).map_err(Into::into) - } - } - - fn _arroy_items_in_store( - &self, - rtxn: &RoTxn, - db: arroy::Database, - store_id: u8, - with_items: F, - ) -> Result - where - F: FnOnce(&RoaringBitmap) -> O, - { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let reader = arroy::Reader::open(rtxn, index, db); - match reader { - Ok(reader) => Ok(with_items(reader.item_ids())), - Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), - Err(err) => Err(err), - } - } - - fn _items_in_store( - &self, - rtxn: &RoTxn, - db: hannoy::Database, - store_id: u8, - with_items: F, - ) -> Result - where - F: FnOnce(&RoaringBitmap) -> O, - { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let reader = hannoy::Reader::open(rtxn, index, db); - match reader { - Ok(reader) => Ok(with_items(reader.item_ids())), - Err(hannoy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), - Err(err) => Err(err), - } - } - - pub fn dimensions(&self, rtxn: &RoTxn) -> crate::Result> { - if self.version_uses_arroy() { - if self.quantized { - Ok(self - .arroy_readers(rtxn, self.arroy_quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } else { - Ok(self - .arroy_readers(rtxn, self.arroy_angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } - } else if self.quantized { - Ok(self - .readers(rtxn, self.quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } else { - Ok(self - .readers(rtxn, self.angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions())) - } - } - - pub fn convert_from_arroy(&self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { - if self.quantized { - let dimensions = self - .arroy_readers(wtxn, self.arroy_quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); - - let Some(dimensions) = dimensions else { return Ok(()) }; - - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); - let mut builder = writer.builder(&mut rng).progress(progress.clone()); - builder.prepare_arroy_conversion(wtxn)?; - builder.build::(wtxn)?; - } - - Ok(()) - } else { - let dimensions = self - .arroy_readers(wtxn, self.arroy_angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); - - let Some(dimensions) = dimensions else { return Ok(()) }; - - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); - let mut builder = writer.builder(&mut rng).progress(progress.clone()); - builder.prepare_arroy_conversion(wtxn)?; - builder.build::(wtxn)?; - } - - Ok(()) - } - } - - #[allow(clippy::too_many_arguments)] - pub fn build_and_quantize( - &mut self, - wtxn: &mut RwTxn, - progress: Progress, - rng: &mut R, - dimension: usize, - quantizing: bool, - hannoy_memory: Option, - cancel: &(impl Fn() -> bool + Sync + Send), - ) -> Result<(), hannoy::Error> { - for index in vector_store_range_for_embedder(self.embedder_index) { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; - } else if writer.is_empty(wtxn)? { - continue; - } - } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - // If we are quantizing the databases, we can't know from meilisearch - // if the db was empty but still contained the wrong metadata, thus we need - // to quantize everything and can't stop early. Since this operation can - // only happens once in the life of an embedder, it's not very performances - // sensitive. - if quantizing && !self.quantized { - let writer = writer.prepare_changing_distance::(wtxn)?; - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; - } else if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; - } else if writer.is_empty(wtxn)? { - continue; - } - } - } - Ok(()) - } - - /// Overwrite all the embeddings associated with the index and item ID. - /// /!\ It won't remove embeddings after the last passed embedding, which can leave stale embeddings. - /// You should call `del_items` on the `item_id` before calling this method. - /// /!\ Cannot insert more than u8::MAX embeddings; after inserting u8::MAX embeddings, all the remaining ones will be silently ignored. - pub fn add_items( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - embeddings: &Embeddings, - ) -> Result<(), hannoy::Error> { - let dimension = embeddings.dimension(); - for (index, vector) in - vector_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) - { - if self.quantized { - hannoy::Writer::new(self.quantized_db(), index, dimension) - .add_item(wtxn, item_id, vector)? - } else { - hannoy::Writer::new(self.angular_db(), index, dimension) - .add_item(wtxn, item_id, vector)? - } - } - Ok(()) - } - - /// Add one document int for this index where we can find an empty spot. - pub fn add_item( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._add_item(wtxn, self.quantized_db(), item_id, vector) - } else { - self._add_item(wtxn, self.angular_db(), item_id, vector) - } - } - - fn _add_item( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result<(), hannoy::Error> { - let dimension = vector.len(); - - for index in vector_store_range_for_embedder(self.embedder_index) { - let writer = hannoy::Writer::new(db, index, dimension); - if !writer.contains_item(wtxn, item_id)? { - writer.add_item(wtxn, item_id, vector)?; - break; - } - } - Ok(()) - } - - /// Add a vector associated with a document in store specified by its id. - /// - /// Any existing vector associated with the document in the store will be replaced by the new vector. - pub fn add_item_in_store( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - store_id: u8, - vector: &[f32], - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._add_item_in_store(wtxn, self.quantized_db(), item_id, store_id, vector) - } else { - self._add_item_in_store(wtxn, self.angular_db(), item_id, store_id, vector) - } - } - - fn _add_item_in_store( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - store_id: u8, - vector: &[f32], - ) -> Result<(), hannoy::Error> { - let dimension = vector.len(); - - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimension); - writer.add_item(wtxn, item_id, vector) - } - - /// Delete all embeddings from a specific `item_id` - pub fn del_items( - &self, - wtxn: &mut RwTxn, - dimension: usize, - item_id: hannoy::ItemId, - ) -> Result<(), hannoy::Error> { - for index in vector_store_range_for_embedder(self.embedder_index) { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - writer.del_item(wtxn, item_id)?; - } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - writer.del_item(wtxn, item_id)?; - } - } - - Ok(()) - } - - /// Removes the item specified by its id from the store specified by its id. - /// - /// Returns whether the item was removed. - /// - /// # Warning - /// - /// - This function will silently fail to remove the item if used against an arroy database that was never built. - pub fn del_item_in_store( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - store_id: u8, - dimensions: usize, - ) -> Result { - if self.quantized { - self._del_item_in_store(wtxn, self.quantized_db(), item_id, store_id, dimensions) - } else { - self._del_item_in_store(wtxn, self.angular_db(), item_id, store_id, dimensions) - } - } - - fn _del_item_in_store( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - store_id: u8, - dimensions: usize, - ) -> Result { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimensions); - writer.del_item(wtxn, item_id) - } - - /// Removes all items from the store specified by its id. - /// - /// # Warning - /// - /// - This function will silently fail to remove the items if used against an arroy database that was never built. - pub fn clear_store( - &self, - wtxn: &mut RwTxn, - store_id: u8, - dimensions: usize, - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._clear_store(wtxn, self.quantized_db(), store_id, dimensions) - } else { - self._clear_store(wtxn, self.angular_db(), store_id, dimensions) - } - } - - fn _clear_store( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - store_id: u8, - dimensions: usize, - ) -> Result<(), hannoy::Error> { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimensions); - writer.clear(wtxn) - } - - /// Delete one item from its value. - pub fn del_item( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result { - if self.quantized { - self._del_item(wtxn, self.quantized_db(), item_id, vector) - } else { - self._del_item(wtxn, self.angular_db(), item_id, vector) - } - } - - fn _del_item( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result { - let dimension = vector.len(); - - for index in vector_store_range_for_embedder(self.embedder_index) { - let writer = hannoy::Writer::new(db, index, dimension); - if writer.contains_item(wtxn, item_id)? { - return writer.del_item(wtxn, item_id); - } - } - Ok(false) - } - - pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), hannoy::Error> { - for index in vector_store_range_for_embedder(self.embedder_index) { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - if writer.is_empty(wtxn)? { - continue; - } - writer.clear(wtxn)?; - } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - if writer.is_empty(wtxn)? { - continue; - } - writer.clear(wtxn)?; - } - } - Ok(()) - } - - pub fn contains_item( - &self, - rtxn: &RoTxn, - dimension: usize, - item: hannoy::ItemId, - ) -> crate::Result { - for index in vector_store_range_for_embedder(self.embedder_index) { - let contains = if self.version_uses_arroy() { - if self.quantized { - let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? - } else { - let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? - } - } else if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? - } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); - if writer.is_empty(rtxn)? { - continue; - } - writer.contains_item(rtxn, item)? - }; - if contains { - return Ok(contains); - } - } - Ok(false) - } - - pub fn nns_by_item( - &self, - rtxn: &RoTxn, - item: ItemId, - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> crate::Result> { - if self.version_uses_arroy() { - if self.quantized { - self._arroy_nns_by_item(rtxn, self.arroy_quantized_db(), item, limit, filter) - .map_err(Into::into) - } else { - self._arroy_nns_by_item(rtxn, self.arroy_angular_db(), item, limit, filter) - .map_err(Into::into) - } - } else if self.quantized { - self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter).map_err(Into::into) - } else { - self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) - } - } - - fn _arroy_nns_by_item( - &self, - rtxn: &RoTxn, - db: arroy::Database, - item: ItemId, - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { - let mut results = Vec::new(); - - for reader in self.arroy_readers(rtxn, db) { - let reader = reader?; - let mut searcher = reader.nns(limit); - if let Some(filter) = filter { - if reader.item_ids().is_disjoint(filter) { - continue; - } - searcher.candidates(filter); - } - - if let Some(mut ret) = searcher.by_item(rtxn, item)? { - results.append(&mut ret); - } - } - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - Ok(results) - } - - fn _nns_by_item( - &self, - rtxn: &RoTxn, - db: hannoy::Database, - item: ItemId, - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> Result, hannoy::Error> { - let mut results = Vec::new(); - - for reader in self.readers(rtxn, db) { - let reader = reader?; - let mut searcher = reader.nns(limit); - searcher.ef_search((limit * 10).max(100)); // TODO find better ef - if let Some(filter) = filter { - searcher.candidates(filter); - } - - if let Some(mut ret) = searcher.by_item(rtxn, item)? { - results.append(&mut ret); - } - } - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - Ok(results) - } - - pub fn nns_by_vector( - &self, - rtxn: &RoTxn, - vector: &[f32], - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> crate::Result> { - if self.version_uses_arroy() { - if self.quantized { - self._arroy_nns_by_vector(rtxn, self.arroy_quantized_db(), vector, limit, filter) - .map_err(Into::into) - } else { - self._arroy_nns_by_vector(rtxn, self.arroy_angular_db(), vector, limit, filter) - .map_err(Into::into) - } - } else if self.quantized { - self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) - .map_err(Into::into) - } else { - self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter).map_err(Into::into) - } - } - - fn _arroy_nns_by_vector( - &self, - rtxn: &RoTxn, - db: arroy::Database, - vector: &[f32], - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> Result, arroy::Error> { - let mut results = Vec::new(); - - for reader in self.arroy_readers(rtxn, db) { - let reader = reader?; - let mut searcher = reader.nns(limit); - if let Some(filter) = filter { - if reader.item_ids().is_disjoint(filter) { - continue; - } - searcher.candidates(filter); - } - - results.append(&mut searcher.by_vector(rtxn, vector)?); - } - - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - - Ok(results) - } - - fn _nns_by_vector( - &self, - rtxn: &RoTxn, - db: hannoy::Database, - vector: &[f32], - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> Result, hannoy::Error> { - let mut results = Vec::new(); - - for reader in self.readers(rtxn, db) { - let reader = reader?; - let mut searcher = reader.nns(limit); - searcher.ef_search((limit * 10).max(100)); // TODO find better ef - if let Some(filter) = filter { - searcher.candidates(filter); - } - - results.append(&mut searcher.by_vector(rtxn, vector)?); - } - - results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - - Ok(results) - } - - pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { - let mut vectors = Vec::new(); - - if self.version_uses_arroy() { - if self.quantized { - for reader in self.arroy_readers(rtxn, self.arroy_quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } else { - for reader in self.arroy_readers(rtxn, self.arroy_angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } - } else if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } else { - for reader in self.readers(rtxn, self.angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } - - Ok(vectors) - } - - fn arroy_angular_db(&self) -> arroy::Database { - self.database.remap_types() - } - - fn arroy_quantized_db(&self) -> arroy::Database { - self.database.remap_types() - } - - fn angular_db(&self) -> hannoy::Database { - self.database.remap_data_type() - } - - fn quantized_db(&self) -> hannoy::Database { - self.database.remap_data_type() - } - - pub fn aggregate_stats( - &self, - rtxn: &RoTxn, - stats: &mut HannoyStats, - ) -> Result<(), hannoy::Error> { - if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - let reader = reader?; - let documents = reader.item_ids(); - stats.documents |= documents; - stats.number_of_embeddings += documents.len(); - } - } else { - for reader in self.readers(rtxn, self.angular_db()) { - let reader = reader?; - let documents = reader.item_ids(); - stats.documents |= documents; - stats.number_of_embeddings += documents.len(); - } - } - - Ok(()) - } -} - -#[derive(Debug, Default, Clone)] -pub struct HannoyStats { - pub number_of_embeddings: u64, - pub documents: RoaringBitmap, -} - -/// One or multiple embeddings stored consecutively in a flat vector. -#[derive(Debug, PartialEq)] -pub struct Embeddings { - data: Vec, - dimension: usize, -} - -impl Embeddings { - /// Declares an empty vector of embeddings of the specified dimensions. - pub fn new(dimension: usize) -> Self { - Self { data: Default::default(), dimension } - } - - /// Declares a vector of embeddings containing a single element. - /// - /// The dimension is inferred from the length of the passed embedding. - pub fn from_single_embedding(embedding: Vec) -> Self { - Self { dimension: embedding.len(), data: embedding } - } - - /// Declares a vector of embeddings from its components. - /// - /// `data.len()` must be a multiple of `dimension`, otherwise an error is returned. - pub fn from_inner(data: Vec, dimension: usize) -> Result> { - let mut this = Self::new(dimension); - this.append(data)?; - Ok(this) - } - - /// Returns the number of embeddings in this vector of embeddings. - pub fn embedding_count(&self) -> usize { - self.data.len() / self.dimension - } - - /// Dimension of a single embedding. - pub fn dimension(&self) -> usize { - self.dimension - } - - /// Deconstructs self into the inner flat vector. - pub fn into_inner(self) -> Vec { - self.data - } - - /// A reference to the inner flat vector. - pub fn as_inner(&self) -> &[F] { - &self.data - } - - /// Iterates over the embeddings contained in the flat vector. - pub fn iter(&self) -> impl Iterator + '_ { - self.data.as_slice().chunks_exact(self.dimension) - } - - /// Push an embedding at the end of the embeddings. - /// - /// If `embedding.len() != self.dimension`, then the push operation fails. - pub fn push(&mut self, mut embedding: Vec) -> Result<(), Vec> { - if embedding.len() != self.dimension { - return Err(embedding); - } - self.data.append(&mut embedding); - Ok(()) - } - - /// Append a flat vector of embeddings at the end of the embeddings. - /// - /// If `embeddings.len() % self.dimension != 0`, then the append operation fails. - pub fn append(&mut self, mut embeddings: Vec) -> Result<(), Vec> { - if embeddings.len() % self.dimension != 0 { - return Err(embeddings); - } - self.data.append(&mut embeddings); - Ok(()) - } -} - -/// An embedder can be used to transform text into embeddings. -#[derive(Debug)] -pub enum Embedder { - /// An embedder based on running local models, fetched from the Hugging Face Hub. - HuggingFace(hf::Embedder), - /// An embedder based on making embedding queries against the OpenAI API. - OpenAi(openai::Embedder), - /// An embedder based on the user providing the embeddings in the documents and queries. - UserProvided(manual::Embedder), - /// An embedder based on making embedding queries against an embedding server. - Ollama(ollama::Embedder), - /// An embedder based on making embedding queries against a generic JSON/REST embedding server. - Rest(rest::Embedder), - /// An embedder composed of an embedder at search time and an embedder at indexing time. - Composite(composite::Embedder), -} - -#[derive(Debug)] -struct EmbeddingCache { - data: Option>>, -} - -impl EmbeddingCache { - const MAX_TEXT_LEN: usize = 2000; - - pub fn new(cap: usize) -> Self { - let data = NonZeroUsize::new(cap).map(lru::LruCache::new).map(Mutex::new); - Self { data } - } - - /// Get the embedding corresponding to `text`, if any is present in the cache. - pub fn get(&self, text: &str) -> Option { - let data = self.data.as_ref()?; - if text.len() > Self::MAX_TEXT_LEN { - return None; - } - let mut cache = data.lock().unwrap(); - - cache.get(text).cloned() - } - - /// Puts a new embedding for the specified `text` - pub fn put(&self, text: String, embedding: Embedding) { - let Some(data) = self.data.as_ref() else { - return; - }; - if text.len() > Self::MAX_TEXT_LEN { - return; - } - tracing::trace!(text, "embedding added to cache"); - - let mut cache = data.lock().unwrap(); - - cache.put(text, embedding); - } -} - -/// Configuration for an embedder. -#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)] -pub struct EmbeddingConfig { - /// Options of the embedder, specific to each kind of embedder - pub embedder_options: EmbedderOptions, - /// Document template - pub prompt: PromptData, - /// If this embedder is binary quantized - pub quantized: Option, - // TODO: add metrics and anything needed -} - -impl EmbeddingConfig { - pub fn quantized(&self) -> bool { - self.quantized.unwrap_or_default() - } -} - -/// Map of runtime embedder data. -#[derive(Clone, Default)] -pub struct RuntimeEmbedders(HashMap>); - -pub struct RuntimeEmbedder { - pub embedder: Arc, - pub document_template: Prompt, - fragments: Vec, - pub is_quantized: bool, -} - -impl RuntimeEmbedder { - pub fn new( - embedder: Arc, - document_template: Prompt, - mut fragments: Vec, - is_quantized: bool, - ) -> Self { - fragments.sort_unstable_by(|left, right| left.name.cmp(&right.name)); - Self { embedder, document_template, fragments, is_quantized } - } - - /// The runtime fragments sorted by name. - pub fn fragments(&self) -> &[RuntimeFragment] { - self.fragments.as_slice() - } -} - -pub struct RuntimeFragment { - pub name: String, - pub id: u8, - pub template: JsonTemplate, -} - -impl RuntimeEmbedders { - /// Create the map from its internal component.s - pub fn new(data: HashMap>) -> Self { - Self(data) - } - - pub fn contains(&self, name: &str) -> bool { - self.0.contains_key(name) - } - - /// Get an embedder configuration and template from its name. - pub fn get(&self, name: &str) -> Option<&Arc> { - self.0.get(name) - } - - pub fn inner_as_ref(&self) -> &HashMap> { - &self.0 - } - - pub fn into_inner(self) -> HashMap> { - self.0 - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl IntoIterator for RuntimeEmbedders { - type Item = (String, Arc); - - type IntoIter = std::collections::hash_map::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -/// Options of an embedder, specific to each kind of embedder. -#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -pub enum EmbedderOptions { - HuggingFace(hf::EmbedderOptions), - OpenAi(openai::EmbedderOptions), - Ollama(ollama::EmbedderOptions), - UserProvided(manual::EmbedderOptions), - Rest(rest::EmbedderOptions), - Composite(composite::EmbedderOptions), -} - -impl EmbedderOptions { - pub fn fragment(&self, name: &str) -> Option<&serde_json::Value> { - match &self { - EmbedderOptions::HuggingFace(_) - | EmbedderOptions::OpenAi(_) - | EmbedderOptions::Ollama(_) - | EmbedderOptions::UserProvided(_) => None, - EmbedderOptions::Rest(embedder_options) => { - embedder_options.indexing_fragments.get(name) - } - EmbedderOptions::Composite(embedder_options) => { - if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index { - embedder_options.indexing_fragments.get(name) - } else { - None - } - } - } - } - - pub fn has_fragments(&self) -> bool { - match &self { - EmbedderOptions::HuggingFace(_) - | EmbedderOptions::OpenAi(_) - | EmbedderOptions::Ollama(_) - | EmbedderOptions::UserProvided(_) => false, - EmbedderOptions::Rest(embedder_options) => { - !embedder_options.indexing_fragments.is_empty() - } - EmbedderOptions::Composite(embedder_options) => { - if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index { - !embedder_options.indexing_fragments.is_empty() - } else { - false - } - } - } - } -} - -impl Default for EmbedderOptions { - fn default() -> Self { - Self::HuggingFace(Default::default()) - } -} - -impl Embedder { - /// Spawns a new embedder built from its options. - pub fn new( - options: EmbedderOptions, - cache_cap: usize, - ) -> std::result::Result { - Ok(match options { - EmbedderOptions::HuggingFace(options) => { - Self::HuggingFace(hf::Embedder::new(options, cache_cap)?) - } - EmbedderOptions::OpenAi(options) => { - Self::OpenAi(openai::Embedder::new(options, cache_cap)?) - } - EmbedderOptions::Ollama(options) => { - Self::Ollama(ollama::Embedder::new(options, cache_cap)?) - } - EmbedderOptions::UserProvided(options) => { - Self::UserProvided(manual::Embedder::new(options)) - } - EmbedderOptions::Rest(options) => Self::Rest(rest::Embedder::new( - options, - cache_cap, - rest::ConfigurationSource::User, - )?), - EmbedderOptions::Composite(options) => { - Self::Composite(composite::Embedder::new(options, cache_cap)?) - } - }) - } - - /// Embed in search context - - #[tracing::instrument(level = "debug", skip_all, target = "search")] - pub fn embed_search( - &self, - query: SearchQuery<'_>, - deadline: Option, - ) -> std::result::Result { - match query { - SearchQuery::Text(text) => self.embed_search_text(text, deadline), - SearchQuery::Media { q, media } => self.embed_search_media(q, media, deadline), - } - } - - pub fn embed_search_text( - &self, - text: &str, - deadline: Option, - ) -> std::result::Result { - if let Some(cache) = self.cache() { - if let Some(embedding) = cache.get(text) { - tracing::trace!(text, "embedding found in cache"); - return Ok(embedding); - } - } - let embedding = match self { - Embedder::HuggingFace(embedder) => embedder.embed_one(text), - Embedder::OpenAi(embedder) => embedder - .embed(&[text], deadline, None)? - .pop() - .ok_or_else(EmbedError::missing_embedding), - Embedder::Ollama(embedder) => embedder - .embed(&[text], deadline, None)? - .pop() - .ok_or_else(EmbedError::missing_embedding), - Embedder::UserProvided(embedder) => embedder.embed_one(text), - Embedder::Rest(embedder) => embedder.embed_one(SearchQuery::Text(text), deadline, None), - Embedder::Composite(embedder) => embedder.search.embed_one(text, deadline, None), - }?; - - if let Some(cache) = self.cache() { - cache.put(text.to_owned(), embedding.clone()); - } - - Ok(embedding) - } - - pub fn embed_search_media( - &self, - q: Option<&str>, - media: Option<&serde_json::Value>, - deadline: Option, - ) -> std::result::Result { - let Embedder::Rest(embedder) = self else { - return Err(EmbedError::rest_media_not_a_rest()); - }; - embedder.embed_one(SearchQuery::Media { q, media }, deadline, None) - } - - /// Embed multiple chunks of texts. - /// - /// Each chunk is composed of one or multiple texts. - pub fn embed_index( - &self, - text_chunks: Vec>, - threads: &ThreadPoolNoAbort, - embedder_stats: &EmbedderStats, - ) -> std::result::Result>, EmbedError> { - match self { - Embedder::HuggingFace(embedder) => embedder.embed_index(text_chunks), - Embedder::OpenAi(embedder) => { - embedder.embed_index(text_chunks, threads, embedder_stats) - } - Embedder::Ollama(embedder) => { - embedder.embed_index(text_chunks, threads, embedder_stats) - } - Embedder::UserProvided(embedder) => embedder.embed_index(text_chunks), - Embedder::Rest(embedder) => embedder.embed_index(text_chunks, threads, embedder_stats), - Embedder::Composite(embedder) => { - embedder.index.embed_index(text_chunks, threads, embedder_stats) - } - } - } - - /// Non-owning variant of [`Self::embed_index`]. - pub fn embed_index_ref( - &self, - texts: &[&str], - threads: &ThreadPoolNoAbort, - embedder_stats: &EmbedderStats, - ) -> std::result::Result, EmbedError> { - match self { - Embedder::HuggingFace(embedder) => embedder.embed_index_ref(texts), - Embedder::OpenAi(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), - Embedder::Ollama(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), - Embedder::UserProvided(embedder) => embedder.embed_index_ref(texts), - Embedder::Rest(embedder) => embedder.embed_index_ref(texts, threads, embedder_stats), - Embedder::Composite(embedder) => { - embedder.index.embed_index_ref(texts, threads, embedder_stats) - } - } - } - - pub fn embed_index_ref_fragments( - &self, - fragments: &[serde_json::Value], - threads: &ThreadPoolNoAbort, - embedder_stats: &EmbedderStats, - ) -> std::result::Result, EmbedError> { - if let Embedder::Rest(embedder) = self { - embedder.embed_index_ref(fragments, threads, embedder_stats) - } else { - let Embedder::Composite(embedder) = self else { - unimplemented!("embedding fragments is only available for rest embedders") - }; - let crate::vector::composite::SubEmbedder::Rest(embedder) = &embedder.index else { - unimplemented!("embedding fragments is only available for rest embedders") - }; - - embedder.embed_index_ref(fragments, threads, embedder_stats) - } - } - - /// Indicates the preferred number of chunks to pass to [`Self::embed_chunks`] - pub fn chunk_count_hint(&self) -> usize { - match self { - Embedder::HuggingFace(embedder) => embedder.chunk_count_hint(), - Embedder::OpenAi(embedder) => embedder.chunk_count_hint(), - Embedder::Ollama(embedder) => embedder.chunk_count_hint(), - Embedder::UserProvided(_) => 100, - Embedder::Rest(embedder) => embedder.chunk_count_hint(), - Embedder::Composite(embedder) => embedder.index.chunk_count_hint(), - } - } - - /// Indicates the preferred number of texts in a single chunk passed to [`Self::embed`] - pub fn prompt_count_in_chunk_hint(&self) -> usize { - match self { - Embedder::HuggingFace(embedder) => embedder.prompt_count_in_chunk_hint(), - Embedder::OpenAi(embedder) => embedder.prompt_count_in_chunk_hint(), - Embedder::Ollama(embedder) => embedder.prompt_count_in_chunk_hint(), - Embedder::UserProvided(_) => 1, - Embedder::Rest(embedder) => embedder.prompt_count_in_chunk_hint(), - Embedder::Composite(embedder) => embedder.index.prompt_count_in_chunk_hint(), - } - } - - /// Indicates the dimensions of a single embedding produced by the embedder. - pub fn dimensions(&self) -> usize { - match self { - Embedder::HuggingFace(embedder) => embedder.dimensions(), - Embedder::OpenAi(embedder) => embedder.dimensions(), - Embedder::Ollama(embedder) => embedder.dimensions(), - Embedder::UserProvided(embedder) => embedder.dimensions(), - Embedder::Rest(embedder) => embedder.dimensions(), - Embedder::Composite(embedder) => embedder.dimensions(), - } - } - - /// An optional distribution used to apply an affine transformation to the similarity score of a document. - pub fn distribution(&self) -> Option { - match self { - Embedder::HuggingFace(embedder) => embedder.distribution(), - Embedder::OpenAi(embedder) => embedder.distribution(), - Embedder::Ollama(embedder) => embedder.distribution(), - Embedder::UserProvided(embedder) => embedder.distribution(), - Embedder::Rest(embedder) => embedder.distribution(), - Embedder::Composite(embedder) => embedder.distribution(), - } - } - - pub fn uses_document_template(&self) -> bool { - match self { - Embedder::HuggingFace(_) - | Embedder::OpenAi(_) - | Embedder::Ollama(_) - | Embedder::Rest(_) => true, - Embedder::UserProvided(_) => false, - Embedder::Composite(embedder) => embedder.index.uses_document_template(), - } - } - - fn cache(&self) -> Option<&EmbeddingCache> { - match self { - Embedder::HuggingFace(embedder) => Some(embedder.cache()), - Embedder::OpenAi(embedder) => Some(embedder.cache()), - Embedder::UserProvided(_) => None, - Embedder::Ollama(embedder) => Some(embedder.cache()), - Embedder::Rest(embedder) => Some(embedder.cache()), - Embedder::Composite(embedder) => embedder.search.cache(), - } - } -} - -#[derive(Clone, Copy)] -pub enum SearchQuery<'a> { - Text(&'a str), - Media { q: Option<&'a str>, media: Option<&'a serde_json::Value> }, -} - -/// Describes the mean and sigma of distribution of embedding similarity in the embedding space. -/// -/// The intended use is to make the similarity score more comparable to the regular ranking score. -/// This allows to correct effects where results are too "packed" around a certain value. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize, ToSchema)] -#[serde(from = "DistributionShiftSerializable")] -#[serde(into = "DistributionShiftSerializable")] -pub struct DistributionShift { - /// Value where the results are "packed". - /// - /// Similarity scores are translated so that they are packed around 0.5 instead - #[schema(value_type = f32)] - pub current_mean: OrderedFloat, - - /// standard deviation of a similarity score. - /// - /// Set below 0.4 to make the results less packed around the mean, and above 0.4 to make them more packed. - #[schema(value_type = f32)] - pub current_sigma: OrderedFloat, -} - -impl Deserr for DistributionShift -where - E: DeserializeError, -{ - fn deserialize_from_value( - value: deserr::Value, - location: deserr::ValuePointerRef<'_>, - ) -> Result { - let value = DistributionShiftSerializable::deserialize_from_value(value, location)?; - if value.mean < 0. || value.mean > 1. { - return Err(deserr::take_cf_content(E::error::( - None, - deserr::ErrorKind::Unexpected { - msg: format!( - "the distribution mean must be in the range [0, 1], got {}", - value.mean - ), - }, - location, - ))); - } - if value.sigma <= 0. || value.sigma > 1. { - return Err(deserr::take_cf_content(E::error::( - None, - deserr::ErrorKind::Unexpected { - msg: format!( - "the distribution sigma must be in the range ]0, 1], got {}", - value.sigma - ), - }, - location, - ))); - } - - Ok(value.into()) - } -} - -#[derive(Serialize, Deserialize, Deserr)] -#[serde(deny_unknown_fields)] -#[deserr(deny_unknown_fields)] -struct DistributionShiftSerializable { - mean: f32, - sigma: f32, -} - -impl From for DistributionShiftSerializable { - fn from( - DistributionShift { - current_mean: OrderedFloat(current_mean), - current_sigma: OrderedFloat(current_sigma), - }: DistributionShift, - ) -> Self { - Self { mean: current_mean, sigma: current_sigma } - } -} - -impl From for DistributionShift { - fn from(DistributionShiftSerializable { mean, sigma }: DistributionShiftSerializable) -> Self { - Self { current_mean: OrderedFloat(mean), current_sigma: OrderedFloat(sigma) } - } -} - -impl DistributionShift { - /// `None` if sigma <= 0. - pub fn new(mean: f32, sigma: f32) -> Option { - if sigma <= 0.0 { - None - } else { - Some(Self { current_mean: OrderedFloat(mean), current_sigma: OrderedFloat(sigma) }) - } - } - - pub fn shift(&self, score: f32) -> f32 { - let current_mean = self.current_mean.0; - let current_sigma = self.current_sigma.0; - // - // We're somewhat abusively mapping the distribution of distances to a gaussian. - // The parameters we're given is the mean and sigma of the native result distribution. - // We're using them to retarget the distribution to a gaussian centered on 0.5 with a sigma of 0.4. - - let target_mean = 0.5; - let target_sigma = 0.4; - - // a^2 sig1^2 = sig2^2 => a^2 = sig2^2 / sig1^2 => a = sig2 / sig1, assuming a, sig1, and sig2 positive. - let factor = target_sigma / current_sigma; - // a*mu1 + b = mu2 => b = mu2 - a*mu1 - let offset = target_mean - (factor * current_mean); - - let mut score = factor * score + offset; - - // clamp the final score in the ]0, 1] interval. - if score <= 0.0 { - score = f32::EPSILON; - } - if score > 1.0 { - score = 1.0; - } - - score - } -} /// Whether CUDA is supported in this version of Meilisearch. pub const fn is_cuda_enabled() -> bool { cfg!(feature = "cuda") } - -fn vector_store_range_for_embedder(embedder_id: u8) -> impl Iterator { - (0..=u8::MAX).map(move |store_id| vector_store_for_embedder(embedder_id, store_id)) -} - -fn vector_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { - let embedder_id = (embedder_id as u16) << 8; - embedder_id | (store_id as u16) -} diff --git a/crates/milli/src/vector/runtime.rs b/crates/milli/src/vector/runtime.rs new file mode 100644 index 000000000..5a653f1b1 --- /dev/null +++ b/crates/milli/src/vector/runtime.rs @@ -0,0 +1,81 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use super::Embedder; +use crate::prompt::Prompt; +use crate::vector::json_template::JsonTemplate; + +/// Map of runtime embedder data. +#[derive(Clone, Default)] +pub struct RuntimeEmbedders(HashMap>); + +pub struct RuntimeEmbedder { + pub embedder: Arc, + pub document_template: Prompt, + fragments: Vec, + pub is_quantized: bool, +} + +impl RuntimeEmbedder { + pub fn new( + embedder: Arc, + document_template: Prompt, + mut fragments: Vec, + is_quantized: bool, + ) -> Self { + fragments.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + Self { embedder, document_template, fragments, is_quantized } + } + + /// The runtime fragments sorted by name. + pub fn fragments(&self) -> &[RuntimeFragment] { + self.fragments.as_slice() + } +} +pub struct RuntimeFragment { + pub name: String, + pub id: u8, + pub template: JsonTemplate, +} + +impl RuntimeEmbedders { + /// Create the map from its internal component.s + pub fn new(data: HashMap>) -> Self { + Self(data) + } + + pub fn contains(&self, name: &str) -> bool { + self.0.contains_key(name) + } + + /// Get an embedder configuration and template from its name. + pub fn get(&self, name: &str) -> Option<&Arc> { + self.0.get(name) + } + + pub fn inner_as_ref(&self) -> &HashMap> { + &self.0 + } + + pub fn into_inner(self) -> HashMap> { + self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl IntoIterator for RuntimeEmbedders { + type Item = (String, Arc); + + type IntoIter = std::collections::hash_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} diff --git a/crates/milli/src/vector/session.rs b/crates/milli/src/vector/session.rs index b582bd840..b7ee7262b 100644 --- a/crates/milli/src/vector/session.rs +++ b/crates/milli/src/vector/session.rs @@ -2,7 +2,8 @@ use bumpalo::collections::Vec as BVec; use bumpalo::Bump; use serde_json::Value; -use super::{EmbedError, Embedder, Embedding}; +use super::error::EmbedError; +use super::{Embedder, Embedding}; use crate::progress::EmbedderStats; use crate::{DocumentId, Result, ThreadPoolNoAbort}; type ExtractorId = u8; diff --git a/crates/milli/src/vector/settings.rs b/crates/milli/src/vector/settings.rs index 1b85dd503..499ab3955 100644 --- a/crates/milli/src/vector/settings.rs +++ b/crates/milli/src/vector/settings.rs @@ -8,12 +8,12 @@ use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; -use super::composite::SubEmbedderOptions; -use super::hf::OverridePooling; -use super::{ollama, openai, DistributionShift, EmbedderOptions}; use crate::prompt::{default_max_bytes, PromptData}; use crate::update::Setting; -use crate::vector::EmbeddingConfig; +use crate::vector::embedder::composite::{self, SubEmbedderOptions}; +use crate::vector::embedder::hf::{self, OverridePooling}; +use crate::vector::embedder::{manual, ollama, openai, rest, EmbedderOptions}; +use crate::vector::{DistributionShift, EmbeddingConfig}; use crate::UserError; #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] @@ -1789,12 +1789,7 @@ pub struct Fragment { impl EmbeddingSettings { fn from_hugging_face( - super::hf::EmbedderOptions { - model, - revision, - distribution, - pooling, - }: super::hf::EmbedderOptions, + hf::EmbedderOptions { model, revision, distribution, pooling }: hf::EmbedderOptions, document_template: Setting, document_template_max_bytes: Setting, quantized: Option, @@ -1822,13 +1817,13 @@ impl EmbeddingSettings { } fn from_openai( - super::openai::EmbedderOptions { + openai::EmbedderOptions { url, api_key, embedding_model, dimensions, distribution, - }: super::openai::EmbedderOptions, + }: openai::EmbedderOptions, document_template: Setting, document_template_max_bytes: Setting, quantized: Option, @@ -1856,13 +1851,13 @@ impl EmbeddingSettings { } fn from_ollama( - super::ollama::EmbedderOptions { - embedding_model, - url, - api_key, - distribution, - dimensions, - }: super::ollama::EmbedderOptions, + ollama::EmbedderOptions { + embedding_model, + url, + api_key, + distribution, + dimensions, + }: ollama::EmbedderOptions, document_template: Setting, document_template_max_bytes: Setting, quantized: Option, @@ -1890,7 +1885,7 @@ impl EmbeddingSettings { } fn from_user_provided( - super::manual::EmbedderOptions { dimensions, distribution }: super::manual::EmbedderOptions, + manual::EmbedderOptions { dimensions, distribution }: manual::EmbedderOptions, quantized: Option, ) -> Self { Self { @@ -1916,7 +1911,7 @@ impl EmbeddingSettings { } fn from_rest( - super::rest::EmbedderOptions { + rest::EmbedderOptions { api_key, dimensions, url, @@ -1926,7 +1921,7 @@ impl EmbeddingSettings { response, distribution, headers, - }: super::rest::EmbedderOptions, + }: rest::EmbedderOptions, document_template: Setting, document_template_max_bytes: Setting, quantized: Option, @@ -2015,37 +2010,36 @@ impl From for EmbeddingSettings { document_template_max_bytes, quantized, ), - super::EmbedderOptions::Composite(super::composite::EmbedderOptions { - search, - index, - }) => Self { - source: Setting::Set(EmbedderSource::Composite), - model: Setting::NotSet, - revision: Setting::NotSet, - pooling: Setting::NotSet, - api_key: Setting::NotSet, - dimensions: Setting::NotSet, - binary_quantized: Setting::some_or_not_set(quantized), - document_template: Setting::NotSet, - document_template_max_bytes: Setting::NotSet, - url: Setting::NotSet, - indexing_fragments: Setting::NotSet, - search_fragments: Setting::NotSet, - request: Setting::NotSet, - response: Setting::NotSet, - headers: Setting::NotSet, - distribution: Setting::some_or_not_set(search.distribution()), - search_embedder: Setting::Set(SubEmbeddingSettings::from_options( - search, - Setting::NotSet, - Setting::NotSet, - )), - indexing_embedder: Setting::Set(SubEmbeddingSettings::from_options( - index, - Setting::Set(prompt.template), - document_template_max_bytes, - )), - }, + super::EmbedderOptions::Composite(composite::EmbedderOptions { search, index }) => { + Self { + source: Setting::Set(EmbedderSource::Composite), + model: Setting::NotSet, + revision: Setting::NotSet, + pooling: Setting::NotSet, + api_key: Setting::NotSet, + dimensions: Setting::NotSet, + binary_quantized: Setting::some_or_not_set(quantized), + document_template: Setting::NotSet, + document_template_max_bytes: Setting::NotSet, + url: Setting::NotSet, + indexing_fragments: Setting::NotSet, + search_fragments: Setting::NotSet, + request: Setting::NotSet, + response: Setting::NotSet, + headers: Setting::NotSet, + distribution: Setting::some_or_not_set(search.distribution()), + search_embedder: Setting::Set(SubEmbeddingSettings::from_options( + search, + Setting::NotSet, + Setting::NotSet, + )), + indexing_embedder: Setting::Set(SubEmbeddingSettings::from_options( + index, + Setting::Set(prompt.template), + document_template_max_bytes, + )), + } + } } } } @@ -2212,7 +2206,7 @@ impl From for EmbeddingConfig { ) .into(), EmbedderSource::Composite => { - super::EmbedderOptions::Composite(super::composite::EmbedderOptions { + super::EmbedderOptions::Composite(composite::EmbedderOptions { // it is important to give the distribution to the search here, as this is from where we'll retrieve it search: SubEmbedderOptions::from_settings( search_embedder.set().unwrap(), @@ -2290,9 +2284,9 @@ impl SubEmbedderOptions { dimensions: Setting, distribution: Setting, ) -> Self { - let mut options = super::openai::EmbedderOptions::with_default_model(None); + let mut options = openai::EmbedderOptions::with_default_model(None); if let Some(model) = model.set() { - if let Some(model) = super::openai::EmbeddingModel::from_name(&model) { + if let Some(model) = openai::EmbeddingModel::from_name(&model) { options.embedding_model = model; } } @@ -2314,7 +2308,7 @@ impl SubEmbedderOptions { pooling: Setting, distribution: Setting, ) -> Self { - let mut options = super::hf::EmbedderOptions::default(); + let mut options = hf::EmbedderOptions::default(); if let Some(model) = model.set() { options.model = model; // Reset the revision if we are setting the model. @@ -2334,10 +2328,7 @@ impl SubEmbedderOptions { SubEmbedderOptions::HuggingFace(options) } fn user_provided(dimensions: usize, distribution: Setting) -> Self { - Self::UserProvided(super::manual::EmbedderOptions { - dimensions, - distribution: distribution.set(), - }) + Self::UserProvided(manual::EmbedderOptions { dimensions, distribution: distribution.set() }) } #[allow(clippy::too_many_arguments)] @@ -2352,7 +2343,7 @@ impl SubEmbedderOptions { dimensions: Setting, distribution: Setting, ) -> Self { - Self::Rest(super::rest::EmbedderOptions { + Self::Rest(rest::EmbedderOptions { api_key: api_key.set(), dimensions: dimensions.set(), url, @@ -2386,11 +2377,7 @@ impl SubEmbedderOptions { distribution: Setting, ) -> Self { let mut options: ollama::EmbedderOptions = - super::ollama::EmbedderOptions::with_default_model( - api_key.set(), - url.set(), - dimensions.set(), - ); + ollama::EmbedderOptions::with_default_model(api_key.set(), url.set(), dimensions.set()); if let Some(model) = model.set() { options.embedding_model = model; } diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs new file mode 100644 index 000000000..72b2985d4 --- /dev/null +++ b/crates/milli/src/vector/store.rs @@ -0,0 +1,775 @@ +use hannoy::distances::{Cosine, Hamming}; +use hannoy::ItemId; +use heed::{RoTxn, RwTxn, Unspecified}; +use ordered_float::OrderedFloat; +use rand::SeedableRng as _; +use roaring::RoaringBitmap; + +use crate::progress::Progress; +use crate::vector::Embeddings; + +const HANNOY_EF_CONSTRUCTION: usize = 125; +const HANNOY_M: usize = 16; +const HANNOY_M0: usize = 32; + +pub struct VectorStore { + version: (u32, u32, u32), + database: hannoy::Database, + embedder_index: u8, + quantized: bool, +} + +impl VectorStore { + pub fn new( + version: (u32, u32, u32), + database: hannoy::Database, + embedder_index: u8, + quantized: bool, + ) -> Self { + Self { version, database, embedder_index, quantized } + } + + pub fn embedder_index(&self) -> u8 { + self.embedder_index + } + + /// Whether we must use the arroy to read the vector store. + pub fn version_uses_arroy(&self) -> bool { + let (major, minor, _patch) = self.version; + major == 1 && minor < 18 + } + + fn arroy_readers<'a, D: arroy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: arroy::Database, + ) -> impl Iterator, arroy::Error>> + 'a { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match arroy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(arroy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) + } + + fn readers<'a, D: hannoy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: hannoy::Database, + ) -> impl Iterator, hannoy::Error>> + 'a { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match hannoy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(hannoy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) + } + + /// The item ids that are present in the store specified by its id. + /// + /// The ids are accessed via a lambda to avoid lifetime shenanigans. + pub fn items_in_store( + &self, + rtxn: &RoTxn, + store_id: u8, + with_items: F, + ) -> crate::Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + if self.version_uses_arroy() { + if self.quantized { + self._arroy_items_in_store(rtxn, self.arroy_quantized_db(), store_id, with_items) + .map_err(Into::into) + } else { + self._arroy_items_in_store(rtxn, self.arroy_angular_db(), store_id, with_items) + .map_err(Into::into) + } + } else if self.quantized { + self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) + .map_err(Into::into) + } else { + self._items_in_store(rtxn, self.angular_db(), store_id, with_items).map_err(Into::into) + } + } + + fn _arroy_items_in_store( + &self, + rtxn: &RoTxn, + db: arroy::Database, + store_id: u8, + with_items: F, + ) -> Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let reader = arroy::Reader::open(rtxn, index, db); + match reader { + Ok(reader) => Ok(with_items(reader.item_ids())), + Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(err) => Err(err), + } + } + + fn _items_in_store( + &self, + rtxn: &RoTxn, + db: hannoy::Database, + store_id: u8, + with_items: F, + ) -> Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let reader = hannoy::Reader::open(rtxn, index, db); + match reader { + Ok(reader) => Ok(with_items(reader.item_ids())), + Err(hannoy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(err) => Err(err), + } + } + + pub fn dimensions(&self, rtxn: &RoTxn) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + Ok(self + .arroy_readers(rtxn, self.arroy_quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } else { + Ok(self + .arroy_readers(rtxn, self.arroy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } + } else if self.quantized { + Ok(self + .readers(rtxn, self.quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } else { + Ok(self + .readers(rtxn, self.angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions())) + } + } + + pub fn convert_from_arroy(&self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { + if self.quantized { + let dimensions = self + .arroy_readers(wtxn, self.arroy_quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) + } else { + let dimensions = self + .arroy_readers(wtxn, self.arroy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) + } + } + + #[allow(clippy::too_many_arguments)] + pub fn build_and_quantize( + &mut self, + wtxn: &mut RwTxn, + progress: Progress, + rng: &mut R, + dimension: usize, + quantizing: bool, + hannoy_memory: Option, + cancel: &(impl Fn() -> bool + Sync + Send), + ) -> Result<(), hannoy::Error> { + for index in vector_store_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if writer.need_build(wtxn)? { + let mut builder = writer.builder(rng).progress(progress.clone()); + builder + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; + } else if writer.is_empty(wtxn)? { + continue; + } + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + // If we are quantizing the databases, we can't know from meilisearch + // if the db was empty but still contained the wrong metadata, thus we need + // to quantize everything and can't stop early. Since this operation can + // only happens once in the life of an embedder, it's not very performances + // sensitive. + if quantizing && !self.quantized { + let writer = writer.prepare_changing_distance::(wtxn)?; + let mut builder = writer.builder(rng).progress(progress.clone()); + builder + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; + } else if writer.need_build(wtxn)? { + let mut builder = writer.builder(rng).progress(progress.clone()); + builder + .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; + } else if writer.is_empty(wtxn)? { + continue; + } + } + } + Ok(()) + } + + /// Overwrite all the embeddings associated with the index and item ID. + /// /!\ It won't remove embeddings after the last passed embedding, which can leave stale embeddings. + /// You should call `del_items` on the `item_id` before calling this method. + /// /!\ Cannot insert more than u8::MAX embeddings; after inserting u8::MAX embeddings, all the remaining ones will be silently ignored. + pub fn add_items( + &self, + wtxn: &mut RwTxn, + item_id: hannoy::ItemId, + embeddings: &Embeddings, + ) -> Result<(), hannoy::Error> { + let dimension = embeddings.dimension(); + for (index, vector) in + vector_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) + { + if self.quantized { + hannoy::Writer::new(self.quantized_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } else { + hannoy::Writer::new(self.angular_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } + } + Ok(()) + } + + /// Add one document int for this index where we can find an empty spot. + pub fn add_item( + &self, + wtxn: &mut RwTxn, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + if self.quantized { + self._add_item(wtxn, self.quantized_db(), item_id, vector) + } else { + self._add_item(wtxn, self.angular_db(), item_id, vector) + } + } + + fn _add_item( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); + if !writer.contains_item(wtxn, item_id)? { + writer.add_item(wtxn, item_id, vector)?; + break; + } + } + Ok(()) + } + + /// Add a vector associated with a document in store specified by its id. + /// + /// Any existing vector associated with the document in the store will be replaced by the new vector. + pub fn add_item_in_store( + &self, + wtxn: &mut RwTxn, + item_id: hannoy::ItemId, + store_id: u8, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + if self.quantized { + self._add_item_in_store(wtxn, self.quantized_db(), item_id, store_id, vector) + } else { + self._add_item_in_store(wtxn, self.angular_db(), item_id, store_id, vector) + } + } + + fn _add_item_in_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + store_id: u8, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + let dimension = vector.len(); + + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimension); + writer.add_item(wtxn, item_id, vector) + } + + /// Delete all embeddings from a specific `item_id` + pub fn del_items( + &self, + wtxn: &mut RwTxn, + dimension: usize, + item_id: hannoy::ItemId, + ) -> Result<(), hannoy::Error> { + for index in vector_store_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + writer.del_item(wtxn, item_id)?; + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + writer.del_item(wtxn, item_id)?; + } + } + + Ok(()) + } + + /// Removes the item specified by its id from the store specified by its id. + /// + /// Returns whether the item was removed. + /// + /// # Warning + /// + /// - This function will silently fail to remove the item if used against an arroy database that was never built. + pub fn del_item_in_store( + &self, + wtxn: &mut RwTxn, + item_id: hannoy::ItemId, + store_id: u8, + dimensions: usize, + ) -> Result { + if self.quantized { + self._del_item_in_store(wtxn, self.quantized_db(), item_id, store_id, dimensions) + } else { + self._del_item_in_store(wtxn, self.angular_db(), item_id, store_id, dimensions) + } + } + + fn _del_item_in_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + store_id: u8, + dimensions: usize, + ) -> Result { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); + writer.del_item(wtxn, item_id) + } + + /// Removes all items from the store specified by its id. + /// + /// # Warning + /// + /// - This function will silently fail to remove the items if used against an arroy database that was never built. + pub fn clear_store( + &self, + wtxn: &mut RwTxn, + store_id: u8, + dimensions: usize, + ) -> Result<(), hannoy::Error> { + if self.quantized { + self._clear_store(wtxn, self.quantized_db(), store_id, dimensions) + } else { + self._clear_store(wtxn, self.angular_db(), store_id, dimensions) + } + } + + fn _clear_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + store_id: u8, + dimensions: usize, + ) -> Result<(), hannoy::Error> { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); + writer.clear(wtxn) + } + + /// Delete one item from its value. + pub fn del_item( + &self, + wtxn: &mut RwTxn, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result { + if self.quantized { + self._del_item(wtxn, self.quantized_db(), item_id, vector) + } else { + self._del_item(wtxn, self.angular_db(), item_id, vector) + } + } + + fn _del_item( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); + if writer.contains_item(wtxn, item_id)? { + return writer.del_item(wtxn, item_id); + } + } + Ok(false) + } + + pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), hannoy::Error> { + for index in vector_store_range_for_embedder(self.embedder_index) { + if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(wtxn)? { + continue; + } + writer.clear(wtxn)?; + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(wtxn)? { + continue; + } + writer.clear(wtxn)?; + } + } + Ok(()) + } + + pub fn contains_item( + &self, + rtxn: &RoTxn, + dimension: usize, + item: hannoy::ItemId, + ) -> crate::Result { + for index in vector_store_range_for_embedder(self.embedder_index) { + let contains = if self.version_uses_arroy() { + if self.quantized { + let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + } else { + let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + } + } else if self.quantized { + let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + } else { + let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + if writer.is_empty(rtxn)? { + continue; + } + writer.contains_item(rtxn, item)? + }; + if contains { + return Ok(contains); + } + } + Ok(false) + } + + pub fn nns_by_item( + &self, + rtxn: &RoTxn, + item: ItemId, + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + self._arroy_nns_by_item(rtxn, self.arroy_quantized_db(), item, limit, filter) + .map_err(Into::into) + } else { + self._arroy_nns_by_item(rtxn, self.arroy_angular_db(), item, limit, filter) + .map_err(Into::into) + } + } else if self.quantized { + self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter).map_err(Into::into) + } else { + self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) + } + } + + fn _arroy_nns_by_item( + &self, + rtxn: &RoTxn, + db: arroy::Database, + item: ItemId, + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.arroy_readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + if reader.item_ids().is_disjoint(filter) { + continue; + } + searcher.candidates(filter); + } + + if let Some(mut ret) = searcher.by_item(rtxn, item)? { + results.append(&mut ret); + } + } + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + Ok(results) + } + + fn _nns_by_item( + &self, + rtxn: &RoTxn, + db: hannoy::Database, + item: ItemId, + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, hannoy::Error> { + let mut results = Vec::new(); + + for reader in self.readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + searcher.ef_search((limit * 10).max(100)); // TODO find better ef + if let Some(filter) = filter { + searcher.candidates(filter); + } + + if let Some(mut ret) = searcher.by_item(rtxn, item)? { + results.append(&mut ret); + } + } + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + Ok(results) + } + + pub fn nns_by_vector( + &self, + rtxn: &RoTxn, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> crate::Result> { + if self.version_uses_arroy() { + if self.quantized { + self._arroy_nns_by_vector(rtxn, self.arroy_quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._arroy_nns_by_vector(rtxn, self.arroy_angular_db(), vector, limit, filter) + .map_err(Into::into) + } + } else if self.quantized { + self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter).map_err(Into::into) + } + } + + fn _arroy_nns_by_vector( + &self, + rtxn: &RoTxn, + db: arroy::Database, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, arroy::Error> { + let mut results = Vec::new(); + + for reader in self.arroy_readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + if let Some(filter) = filter { + if reader.item_ids().is_disjoint(filter) { + continue; + } + searcher.candidates(filter); + } + + results.append(&mut searcher.by_vector(rtxn, vector)?); + } + + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + + Ok(results) + } + + fn _nns_by_vector( + &self, + rtxn: &RoTxn, + db: hannoy::Database, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> Result, hannoy::Error> { + let mut results = Vec::new(); + + for reader in self.readers(rtxn, db) { + let reader = reader?; + let mut searcher = reader.nns(limit); + searcher.ef_search((limit * 10).max(100)); // TODO find better ef + if let Some(filter) = filter { + searcher.candidates(filter); + } + + results.append(&mut searcher.by_vector(rtxn, vector)?); + } + + results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + + Ok(results) + } + + pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { + let mut vectors = Vec::new(); + + if self.version_uses_arroy() { + if self.quantized { + for reader in self.arroy_readers(rtxn, self.arroy_quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self.arroy_readers(rtxn, self.arroy_angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } + } else if self.quantized { + for reader in self.readers(rtxn, self.quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self.readers(rtxn, self.angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } + + Ok(vectors) + } + + fn arroy_angular_db(&self) -> arroy::Database { + self.database.remap_types() + } + + fn arroy_quantized_db(&self) -> arroy::Database { + self.database.remap_types() + } + + fn angular_db(&self) -> hannoy::Database { + self.database.remap_data_type() + } + + fn quantized_db(&self) -> hannoy::Database { + self.database.remap_data_type() + } + + pub fn aggregate_stats( + &self, + rtxn: &RoTxn, + stats: &mut HannoyStats, + ) -> Result<(), hannoy::Error> { + if self.quantized { + for reader in self.readers(rtxn, self.quantized_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } else { + for reader in self.readers(rtxn, self.angular_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } + + Ok(()) + } +} + +#[derive(Debug, Default, Clone)] +pub struct HannoyStats { + pub number_of_embeddings: u64, + pub documents: RoaringBitmap, +} + +fn vector_store_range_for_embedder(embedder_id: u8) -> impl Iterator { + (0..=u8::MAX).map(move |store_id| vector_store_for_embedder(embedder_id, store_id)) +} + +fn vector_store_for_embedder(embedder_id: u8, store_id: u8) -> u16 { + let embedder_id = (embedder_id as u16) << 8; + embedder_id | (store_id as u16) +} From da6fffdf6dea93e142500fb6c3a3843cab41873e Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 26 Aug 2025 17:49:56 +0200 Subject: [PATCH 33/62] Switch from version to backend selector --- crates/milli/src/index.rs | 42 ++++++++++++++++--- .../milli/src/search/facet/filter_vector.rs | 4 +- crates/milli/src/search/new/vector_sort.rs | 9 ++-- crates/milli/src/search/similar.rs | 10 ++--- .../milli/src/update/index_documents/mod.rs | 12 ++---- .../src/update/index_documents/transform.rs | 6 +-- .../src/update/index_documents/typed_chunk.rs | 10 ++--- crates/milli/src/update/new/indexer/mod.rs | 30 ++++--------- .../milli/src/update/new/vector_document.rs | 4 +- crates/milli/src/update/upgrade/new_hannoy.rs | 5 ++- crates/milli/src/vector/mod.rs | 2 +- crates/milli/src/vector/store.rs | 32 +++++++------- 12 files changed, 88 insertions(+), 78 deletions(-) diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index b9210fa60..f8210abfa 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -31,7 +31,7 @@ use crate::prompt::PromptData; use crate::proximity::ProximityPrecision; use crate::update::new::StdResult; use crate::vector::db::IndexEmbeddingConfigs; -use crate::vector::{Embedding, HannoyStats, VectorStore}; +use crate::vector::{Embedding, HannoyStats, VectorStore, VectorStoreBackend}; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldIdWordCountCodec, @@ -87,6 +87,7 @@ pub mod main_key { pub const DOCUMENTS_STATS: &str = "documents_stats"; pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms"; pub const CHAT: &str = "chat"; + pub const VECTOR_STORE_BACKEND: &str = "vector_store_backend"; } pub mod db_name { @@ -454,6 +455,35 @@ impl Index { self.main.remap_types::().get(rtxn, main_key::VERSION_KEY) } + /* vector store */ + /// Writes the vector store + pub(crate) fn put_vector_store( + &self, + wtxn: &mut RwTxn<'_>, + backend: VectorStoreBackend, + ) -> Result<()> { + Ok(self.main.remap_types::>().put( + wtxn, + main_key::VECTOR_STORE_BACKEND, + &backend, + )?) + } + + pub(crate) fn get_vector_store(&self, rtxn: &RoTxn<'_>) -> Result { + Ok(self + .main + .remap_types::>() + .get(rtxn, main_key::VECTOR_STORE_BACKEND)? + .unwrap_or_default()) + } + + pub(crate) fn delete_vector_store(&self, wtxn: &mut RwTxn<'_>) -> Result { + Ok(self + .main + .remap_types::>() + .delete(wtxn, main_key::VECTOR_STORE_BACKEND)?) + } + /* documents ids */ /// Writes the documents ids that corresponds to the user-ids-documents-ids FST. @@ -1769,12 +1799,13 @@ impl Index { ) -> Result> { let mut res = BTreeMap::new(); let embedders = self.embedding_configs(); - let index_version = self.get_version(rtxn)?.unwrap(); + let backend = self.get_vector_store(rtxn)?; + for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); let has_fragments = config.config.embedder_options.has_fragments(); let reader = VectorStore::new( - index_version, + backend, self.vector_store, embedder_info.embedder_id, config.config.quantized(), @@ -1797,11 +1828,12 @@ impl Index { pub fn hannoy_stats(&self, rtxn: &RoTxn<'_>) -> Result { let mut stats = HannoyStats::default(); let embedding_configs = self.embedding_configs(); - let index_version = self.get_version(rtxn)?.unwrap(); + let backend = self.get_vector_store(rtxn)?; + for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); let reader = VectorStore::new( - index_version, + backend, self.vector_store, embedder_id, config.config.quantized(), diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 62303c622..d8e007760 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -82,7 +82,7 @@ fn evaluate_inner( embedding_configs: &[IndexEmbeddingConfig], filter: &VectorFilter<'_>, ) -> crate::Result { - let index_version = index.get_version(rtxn)?.unwrap(); + let backend = index.get_vector_store(rtxn)?; let embedder_name = embedder.value(); let available_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); @@ -98,7 +98,7 @@ fn evaluate_inner( .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; let vector_store = VectorStore::new( - index_version, + backend, index.vector_store, embedder_info.embedder_id, embedding_config.config.quantized(), diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index fce3340c5..ecd61e322 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -54,14 +54,11 @@ impl VectorSort { vector_candidates: &RoaringBitmap, ) -> Result<()> { let target = &self.target; + let backend = ctx.index.get_vector_store(ctx.txn)?; let before = Instant::now(); - let reader = VectorStore::new( - ctx.index.get_version(ctx.txn)?.unwrap(), - ctx.index.vector_store, - self.embedder_index, - self.quantized, - ); + let reader = + VectorStore::new(backend, ctx.index.vector_store, self.embedder_index, self.quantized); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index d4b45cd4e..fb2e8103d 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -72,12 +72,10 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let reader = VectorStore::new( - self.index.get_version(self.rtxn)?.unwrap(), - self.index.vector_store, - embedder_index, - self.quantized, - ); + let backend = self.index.get_vector_store(self.rtxn)?; + + let reader = + VectorStore::new(backend, self.index.vector_store, embedder_index, self.quantized); let results = reader.nns_by_item( self.rtxn, self.id, diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 465a40f42..686da865c 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -485,7 +485,7 @@ where // If an embedder wasn't used in the typedchunk but must be binary quantized // we should insert it in `dimension` - let index_version = self.index.get_version(self.wtxn)?.unwrap(); + let backend = self.index.get_vector_store(self.wtxn)?; for (name, action) in settings_diff.embedding_config_updates.iter() { if action.is_being_quantized && !dimension.contains_key(name.as_str()) { let index = self.index.embedding_configs().embedder_id(self.wtxn, name)?.ok_or( @@ -494,12 +494,8 @@ where key: None, }, )?; - let reader = VectorStore::new( - index_version, - self.index.vector_store, - index, - action.was_quantized, - ); + let reader = + VectorStore::new(backend, self.index.vector_store, index, action.was_quantized); let Some(dim) = reader.dimensions(self.wtxn)? else { continue; }; @@ -529,7 +525,7 @@ where pool.install(|| { let mut writer = - VectorStore::new(index_version, vector_hannoy, embedder_index, was_quantized); + VectorStore::new(backend, vector_hannoy, embedder_index, was_quantized); writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index e0d8b82f8..e4bd49434 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -834,7 +834,7 @@ impl<'a, 'i> Transform<'a, 'i> { None }; - let index_version = self.index.get_version(wtxn)?.unwrap(); + let backend = self.index.get_vector_store(wtxn)?; let readers: BTreeMap<&str, (VectorStore, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() @@ -843,7 +843,7 @@ impl<'a, 'i> Transform<'a, 'i> { action.write_back() { let reader = VectorStore::new( - index_version, + backend, self.index.vector_store, *embedder_id, action.was_quantized, @@ -949,7 +949,7 @@ impl<'a, 'i> Transform<'a, 'i> { continue; }; let hannoy = VectorStore::new( - index_version, + backend, self.index.vector_store, infos.embedder_id, was_quantized, diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index 4efe6bde2..228dc4797 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -619,7 +619,7 @@ pub(crate) fn write_typed_chunk_into_index( let _entered = span.enter(); let embedders = index.embedding_configs(); - let index_version = index.get_version(wtxn)?.unwrap(); + let backend = index.get_vector_store(wtxn)?; let mut remove_vectors_builder = MergerBuilder::new(KeepFirst); let mut manual_vectors_builder = MergerBuilder::new(KeepFirst); @@ -678,12 +678,8 @@ pub(crate) fn write_typed_chunk_into_index( .get(&embedder_name) .is_some_and(|conf| conf.is_quantized); // FIXME: allow customizing distance - let writer = VectorStore::new( - index_version, - index.vector_store, - infos.embedder_id, - binary_quantized, - ); + let writer = + VectorStore::new(backend, index.vector_store, infos.embedder_id, binary_quantized); // remove vectors for docids we want them removed let merger = remove_vectors_builder.build(); diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 71402aae9..0547aa155 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -131,7 +131,7 @@ where let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map); let vector_arroy = index.vector_store; - let index_version = index.get_version(wtxn)?.unwrap(); + let backend = index.get_vector_store(wtxn)?; let hannoy_writers: Result> = embedders .inner_as_ref() .iter() @@ -145,12 +145,8 @@ where })?; let dimensions = runtime.embedder.dimensions(); - let writer = VectorStore::new( - index_version, - vector_arroy, - embedder_index, - runtime.is_quantized, - ); + let writer = + VectorStore::new(backend, vector_arroy, embedder_index, runtime.is_quantized); Ok(( embedder_index, @@ -352,7 +348,7 @@ fn hannoy_writers_from_embedder_actions<'indexer>( index_embedder_category_ids: &'indexer std::collections::HashMap, ) -> Result> { let vector_arroy = index.vector_store; - let index_version = index.get_version(rtxn)?.unwrap(); + let backend = index.get_vector_store(rtxn)?; embedders .inner_as_ref() @@ -371,7 +367,7 @@ fn hannoy_writers_from_embedder_actions<'indexer>( ))); }; let writer = VectorStore::new( - index_version, + backend, vector_arroy, embedder_category_id, action.was_quantized, @@ -394,16 +390,13 @@ fn delete_old_embedders_and_fragments( where SD: SettingsDelta, { + let backend = index.get_vector_store(wtxn)?; for action in settings_delta.embedder_actions().values() { let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; }; - let reader = VectorStore::new( - index.get_version(wtxn)?.unwrap(), - index.vector_store, - *embedder_id, - action.was_quantized, - ); + let reader = + VectorStore::new(backend, index.vector_store, *embedder_id, action.was_quantized); let Some(dimensions) = reader.dimensions(wtxn)? else { continue; }; @@ -419,12 +412,7 @@ where let Some(infos) = index.embedding_configs().embedder_info(wtxn, embedder_name)? else { continue; }; - let arroy = VectorStore::new( - index.get_version(wtxn)?.unwrap(), - index.vector_store, - infos.embedder_id, - was_quantized, - ); + let arroy = VectorStore::new(backend, index.vector_store, infos.embedder_id, was_quantized); let Some(dimensions) = arroy.dimensions(wtxn)? else { continue; }; diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index 76639ad31..bff407b9f 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -120,9 +120,9 @@ impl<'t> VectorDocumentFromDb<'t> { config: &IndexEmbeddingConfig, status: &EmbeddingStatus, ) -> Result> { - let index_version = self.index.get_version(self.rtxn)?.unwrap(); + let backend = self.index.get_vector_store(self.rtxn)?; let reader = VectorStore::new( - index_version, + backend, self.index.vector_store, embedder_id, config.config.quantized(), diff --git a/crates/milli/src/update/upgrade/new_hannoy.rs b/crates/milli/src/update/upgrade/new_hannoy.rs index f4e446335..29dad98cb 100644 --- a/crates/milli/src/update/upgrade/new_hannoy.rs +++ b/crates/milli/src/update/upgrade/new_hannoy.rs @@ -17,13 +17,14 @@ impl UpgradeIndex for Latest_V1_18_New_Hannoy { progress: Progress, ) -> Result { let embedding_configs = index.embedding_configs(); - let index_version = index.get_version(wtxn)?.unwrap(); + let backend = index.get_vector_store(wtxn)?; for config in embedding_configs.embedding_configs(wtxn)? { // TODO use the embedder name to display progress + /// REMOVE THIS FILE, IMPLEMENT CONVERSION AS A SETTING CHANGE let quantized = config.config.quantized(); let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); let vector_store = - VectorStore::new(index_version, index.vector_store, embedder_id, quantized); + VectorStore::new(backend, index.vector_store, embedder_id, quantized); vector_store.convert_from_arroy(wtxn, progress.clone())?; } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f9b62d7d8..be747f459 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -19,7 +19,7 @@ pub use distribution::DistributionShift; pub use embedder::{Embedder, EmbedderOptions, EmbeddingConfig, SearchQuery}; pub use embeddings::Embeddings; pub use runtime::{RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment}; -pub use store::{HannoyStats, VectorStore}; +pub use store::{HannoyStats, VectorStore, VectorStoreBackend}; pub const REQUEST_PARALLELISM: usize = 40; diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 72b2985d4..7ced91201 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -4,6 +4,7 @@ use heed::{RoTxn, RwTxn, Unspecified}; use ordered_float::OrderedFloat; use rand::SeedableRng as _; use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; use crate::progress::Progress; use crate::vector::Embeddings; @@ -12,8 +13,15 @@ const HANNOY_EF_CONSTRUCTION: usize = 125; const HANNOY_M: usize = 16; const HANNOY_M0: usize = 32; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum VectorStoreBackend { + #[default] + Arroy, + Hannoy, +} + pub struct VectorStore { - version: (u32, u32, u32), + backend: VectorStoreBackend, database: hannoy::Database, embedder_index: u8, quantized: bool, @@ -21,24 +29,18 @@ pub struct VectorStore { impl VectorStore { pub fn new( - version: (u32, u32, u32), + backend: VectorStoreBackend, database: hannoy::Database, embedder_index: u8, quantized: bool, ) -> Self { - Self { version, database, embedder_index, quantized } + Self { backend, database, embedder_index, quantized } } pub fn embedder_index(&self) -> u8 { self.embedder_index } - /// Whether we must use the arroy to read the vector store. - pub fn version_uses_arroy(&self) -> bool { - let (major, minor, _patch) = self.version; - major == 1 && minor < 18 - } - fn arroy_readers<'a, D: arroy::Distance>( &'a self, rtxn: &'a RoTxn<'a>, @@ -87,7 +89,7 @@ impl VectorStore { where F: FnOnce(&RoaringBitmap) -> O, { - if self.version_uses_arroy() { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { self._arroy_items_in_store(rtxn, self.arroy_quantized_db(), store_id, with_items) .map_err(Into::into) @@ -142,7 +144,7 @@ impl VectorStore { } pub fn dimensions(&self, rtxn: &RoTxn) -> crate::Result> { - if self.version_uses_arroy() { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { Ok(self .arroy_readers(rtxn, self.arroy_quantized_db()) @@ -497,7 +499,7 @@ impl VectorStore { item: hannoy::ItemId, ) -> crate::Result { for index in vector_store_range_for_embedder(self.embedder_index) { - let contains = if self.version_uses_arroy() { + let contains = if self.backend == VectorStoreBackend::Arroy { if self.quantized { let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimension); if writer.is_empty(rtxn)? { @@ -538,7 +540,7 @@ impl VectorStore { limit: usize, filter: Option<&RoaringBitmap>, ) -> crate::Result> { - if self.version_uses_arroy() { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { self._arroy_nns_by_item(rtxn, self.arroy_quantized_db(), item, limit, filter) .map_err(Into::into) @@ -614,7 +616,7 @@ impl VectorStore { limit: usize, filter: Option<&RoaringBitmap>, ) -> crate::Result> { - if self.version_uses_arroy() { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { self._arroy_nns_by_vector(rtxn, self.arroy_quantized_db(), vector, limit, filter) .map_err(Into::into) @@ -687,7 +689,7 @@ impl VectorStore { pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { let mut vectors = Vec::new(); - if self.version_uses_arroy() { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { for reader in self.arroy_readers(rtxn, self.arroy_quantized_db()) { if let Some(vec) = reader?.item_vector(rtxn, item_id)? { From b2f2807a9494a85077dda43abdfc903d86f32bd8 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 28 Aug 2025 14:43:04 +0200 Subject: [PATCH 34/62] Integrate arroy with conversion capabilities --- Cargo.lock | 4 +- crates/milli/Cargo.toml | 8 +- crates/milli/src/progress.rs | 51 ++++++++ crates/milli/src/update/upgrade/new_hannoy.rs | 4 +- crates/milli/src/vector/store.rs | 110 +++++++++++++----- 5 files changed, 141 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ef2372f7..8fec0620c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,9 +444,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arroy" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08e6111f351d004bd13e95ab540721272136fd3218b39d3ec95a2ea1c4e6a0a6" +checksum = "8578a72223dfa13dfd9fc144d15260d134361789ebdea9b16e85a511edc73c7d" dependencies = [ "bytemuck", "byteorder", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index e90d96500..94ac1a09f 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -87,7 +87,7 @@ rhai = { version = "1.22.2", features = [ "no_time", "sync", ] } -arroy = "0.6.1" +arroy = "0.6.3" hannoy = "0.0.4" rand = "0.8.5" tracing = "0.1.41" @@ -111,7 +111,11 @@ utoipa = { version = "5.4.0", features = [ "openapi_extensions", ] } lru = "0.14.0" -twox-hash = { version = "2.1.1", default-features = false, features = ["std", "xxhash3_64", "xxhash64"] } +twox-hash = { version = "2.1.1", default-features = false, features = [ + "std", + "xxhash3_64", + "xxhash64", +] } [dev-dependencies] mimalloc = { version = "0.1.47", default-features = false } diff --git a/crates/milli/src/progress.rs b/crates/milli/src/progress.rs index eb309b0b0..2aab11f05 100644 --- a/crates/milli/src/progress.rs +++ b/crates/milli/src/progress.rs @@ -5,6 +5,7 @@ use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; +use enum_iterator::Sequence as _; use indexmap::IndexMap; use itertools::Itertools; use serde::Serialize; @@ -95,6 +96,14 @@ impl Progress { durations.drain(..).map(|(name, duration)| (name, format!("{duration:.2?}"))).collect() } + + // TODO: ideally we should expose the progress in a way that let arroy use it directly + pub(crate) fn update_progress_from_arroy(&self, progress: arroy::WriterProgress) { + self.update_progress(progress.main); + if let Some(sub) = progress.sub { + self.update_progress(sub); + } + } } /// Generate the names associated with the durations and push them. @@ -291,3 +300,45 @@ impl Step for Compat { self.0.total().try_into().unwrap_or(u32::MAX) } } + +impl Step for arroy::MainStep { + fn name(&self) -> Cow<'static, str> { + match self { + arroy::MainStep::PreProcessingTheItems => "pre processing the items", + arroy::MainStep::WritingTheDescendantsAndMetadata => { + "writing the descendants and metadata" + } + arroy::MainStep::RetrieveTheUpdatedItems => "retrieve the updated items", + arroy::MainStep::RetrievingTheTreeAndItemNodes => "retrieving the tree and item nodes", + arroy::MainStep::UpdatingTheTrees => "updating the trees", + arroy::MainStep::CreateNewTrees => "create new trees", + arroy::MainStep::WritingNodesToDatabase => "writing nodes to database", + arroy::MainStep::DeleteExtraneousTrees => "delete extraneous trees", + arroy::MainStep::WriteTheMetadata => "write the metadata", + arroy::MainStep::ConvertingHannoyToArroy => "converting hannoy to arroy", + } + .into() + } + + fn current(&self) -> u32 { + *self as u32 + } + + fn total(&self) -> u32 { + Self::CARDINALITY as u32 + } +} + +impl Step for arroy::SubStep { + fn name(&self) -> Cow<'static, str> { + self.unit.into() + } + + fn current(&self) -> u32 { + self.current.load(Ordering::Relaxed) + } + + fn total(&self) -> u32 { + self.max + } +} diff --git a/crates/milli/src/update/upgrade/new_hannoy.rs b/crates/milli/src/update/upgrade/new_hannoy.rs index 29dad98cb..097ec7a7e 100644 --- a/crates/milli/src/update/upgrade/new_hannoy.rs +++ b/crates/milli/src/update/upgrade/new_hannoy.rs @@ -23,9 +23,9 @@ impl UpgradeIndex for Latest_V1_18_New_Hannoy { /// REMOVE THIS FILE, IMPLEMENT CONVERSION AS A SETTING CHANGE let quantized = config.config.quantized(); let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); - let vector_store = + let mut vector_store = VectorStore::new(backend, index.vector_store, embedder_id, quantized); - vector_store.convert_from_arroy(wtxn, progress.clone())?; + vector_store.change_backend(wtxn, progress.clone())?; } Ok(false) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 7ced91201..cbc11aa77 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -173,43 +173,93 @@ impl VectorStore { } } - pub fn convert_from_arroy(&self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { - if self.quantized { - let dimensions = self - .arroy_readers(wtxn, self.arroy_quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); + pub fn change_backend(&mut self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { + if self.backend == VectorStoreBackend::Arroy { + self.backend = VectorStoreBackend::Hannoy; + if self.quantized { + let dimensions = self + .arroy_readers(wtxn, self.arroy_quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); - let Some(dimensions) = dimensions else { return Ok(()) }; + let Some(dimensions) = dimensions else { return Ok(()) }; - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); - let mut builder = writer.builder(&mut rng).progress(progress.clone()); - builder.prepare_arroy_conversion(wtxn)?; - builder.build::(wtxn)?; + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) + } else { + let dimensions = self + .arroy_readers(wtxn, self.arroy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); + let mut builder = writer.builder(&mut rng).progress(progress.clone()); + builder.prepare_arroy_conversion(wtxn)?; + builder.build::(wtxn)?; + } + + Ok(()) } - - Ok(()) } else { - let dimensions = self - .arroy_readers(wtxn, self.arroy_angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); + self.backend = VectorStoreBackend::Arroy; - let Some(dimensions) = dimensions else { return Ok(()) }; + if self.quantized { + /// FIXME: make sure that all of the functions in the vector store can work both in arroy and hannoy + /// (the existing implementation was assuming that only reads could happen in arroy) + /// bonus points for making it harder to forget switching on arroy on hannoy when modifying the code + let dimensions = self + .readers(wtxn, self.quantized_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); - let mut builder = writer.builder(&mut rng).progress(progress.clone()); - builder.prepare_arroy_conversion(wtxn)?; - builder.build::(wtxn)?; + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimensions); + let mut builder = writer.builder(&mut rng); + let builder = + builder.progress(|step| progress.update_progress_from_arroy(step)); + builder.prepare_hannoy_conversion(wtxn)?; + builder.build(wtxn)?; + } + + Ok(()) + } else { + let dimensions = self + .readers(wtxn, self.angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); + + let Some(dimensions) = dimensions else { return Ok(()) }; + + for index in vector_store_range_for_embedder(self.embedder_index) { + let mut rng = rand::rngs::StdRng::from_entropy(); + let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimensions); + let mut builder = writer.builder(&mut rng); + let builder = + builder.progress(|step| progress.update_progress_from_arroy(step)); + builder.prepare_hannoy_conversion(wtxn)?; + builder.build(wtxn)?; + } + + Ok(()) } - - Ok(()) } } From 5fc7872ab316d8a6a9799e27769ef4bd833f23cc Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 28 Aug 2025 16:32:47 +0200 Subject: [PATCH 35/62] Make sure the vector store works with both arroy and hannoy --- crates/milli/src/vector/store.rs | 905 ++++++++++++++++++++----------- 1 file changed, 588 insertions(+), 317 deletions(-) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index cbc11aa77..c39596da2 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -28,6 +28,8 @@ pub struct VectorStore { } impl VectorStore { + // backend-independent public functions + pub fn new( backend: VectorStoreBackend, database: hannoy::Database, @@ -41,41 +43,7 @@ impl VectorStore { self.embedder_index } - fn arroy_readers<'a, D: arroy::Distance>( - &'a self, - rtxn: &'a RoTxn<'a>, - db: arroy::Database, - ) -> impl Iterator, arroy::Error>> + 'a { - vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { - match arroy::Reader::open(rtxn, index, db) { - Ok(reader) => match reader.is_empty(rtxn) { - Ok(false) => Some(Ok(reader)), - Ok(true) => None, - Err(e) => Some(Err(e)), - }, - Err(arroy::Error::MissingMetadata(_)) => None, - Err(e) => Some(Err(e)), - } - }) - } - - fn readers<'a, D: hannoy::Distance>( - &'a self, - rtxn: &'a RoTxn<'a>, - db: hannoy::Database, - ) -> impl Iterator, hannoy::Error>> + 'a { - vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { - match hannoy::Reader::open(rtxn, index, db) { - Ok(reader) => match reader.is_empty(rtxn) { - Ok(false) => Some(Ok(reader)), - Ok(true) => None, - Err(e) => Some(Err(e)), - }, - Err(hannoy::Error::MissingMetadata(_)) => None, - Err(e) => Some(Err(e)), - } - }) - } + // backend-dependent public functions /// The item ids that are present in the store specified by its id. /// @@ -91,55 +59,18 @@ impl VectorStore { { if self.backend == VectorStoreBackend::Arroy { if self.quantized { - self._arroy_items_in_store(rtxn, self.arroy_quantized_db(), store_id, with_items) + self._arroy_items_in_store(rtxn, self._arroy_quantized_db(), store_id, with_items) .map_err(Into::into) } else { - self._arroy_items_in_store(rtxn, self.arroy_angular_db(), store_id, with_items) + self._arroy_items_in_store(rtxn, self._arroy_angular_db(), store_id, with_items) .map_err(Into::into) } } else if self.quantized { - self._items_in_store(rtxn, self.quantized_db(), store_id, with_items) + self._hannoy_items_in_store(rtxn, self._hannoy_quantized_db(), store_id, with_items) .map_err(Into::into) } else { - self._items_in_store(rtxn, self.angular_db(), store_id, with_items).map_err(Into::into) - } - } - - fn _arroy_items_in_store( - &self, - rtxn: &RoTxn, - db: arroy::Database, - store_id: u8, - with_items: F, - ) -> Result - where - F: FnOnce(&RoaringBitmap) -> O, - { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let reader = arroy::Reader::open(rtxn, index, db); - match reader { - Ok(reader) => Ok(with_items(reader.item_ids())), - Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), - Err(err) => Err(err), - } - } - - fn _items_in_store( - &self, - rtxn: &RoTxn, - db: hannoy::Database, - store_id: u8, - with_items: F, - ) -> Result - where - F: FnOnce(&RoaringBitmap) -> O, - { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let reader = hannoy::Reader::open(rtxn, index, db); - match reader { - Ok(reader) => Ok(with_items(reader.item_ids())), - Err(hannoy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), - Err(err) => Err(err), + self._hannoy_items_in_store(rtxn, self._hannoy_angular_db(), store_id, with_items) + .map_err(Into::into) } } @@ -147,26 +78,26 @@ impl VectorStore { if self.backend == VectorStoreBackend::Arroy { if self.quantized { Ok(self - .arroy_readers(rtxn, self.arroy_quantized_db()) + ._arroy_readers(rtxn, self._arroy_quantized_db()) .next() .transpose()? .map(|reader| reader.dimensions())) } else { Ok(self - .arroy_readers(rtxn, self.arroy_angular_db()) + ._arroy_readers(rtxn, self._arroy_angular_db()) .next() .transpose()? .map(|reader| reader.dimensions())) } } else if self.quantized { Ok(self - .readers(rtxn, self.quantized_db()) + ._hannoy_readers(rtxn, self._hannoy_quantized_db()) .next() .transpose()? .map(|reader| reader.dimensions())) } else { Ok(self - .readers(rtxn, self.angular_db()) + ._hannoy_readers(rtxn, self._hannoy_angular_db()) .next() .transpose()? .map(|reader| reader.dimensions())) @@ -178,7 +109,7 @@ impl VectorStore { self.backend = VectorStoreBackend::Hannoy; if self.quantized { let dimensions = self - .arroy_readers(wtxn, self.arroy_quantized_db()) + ._arroy_readers(wtxn, self._arroy_quantized_db()) .next() .transpose()? .map(|reader| reader.dimensions()); @@ -187,7 +118,8 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.quantized_db(), index, dimensions); + let writer = + hannoy::Writer::new(self._hannoy_quantized_db(), index, dimensions); let mut builder = writer.builder(&mut rng).progress(progress.clone()); builder.prepare_arroy_conversion(wtxn)?; builder.build::(wtxn)?; @@ -196,7 +128,7 @@ impl VectorStore { Ok(()) } else { let dimensions = self - .arroy_readers(wtxn, self.arroy_angular_db()) + ._arroy_readers(wtxn, self._arroy_angular_db()) .next() .transpose()? .map(|reader| reader.dimensions()); @@ -205,7 +137,7 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = hannoy::Writer::new(self.angular_db(), index, dimensions); + let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimensions); let mut builder = writer.builder(&mut rng).progress(progress.clone()); builder.prepare_arroy_conversion(wtxn)?; builder.build::(wtxn)?; @@ -217,11 +149,8 @@ impl VectorStore { self.backend = VectorStoreBackend::Arroy; if self.quantized { - /// FIXME: make sure that all of the functions in the vector store can work both in arroy and hannoy - /// (the existing implementation was assuming that only reads could happen in arroy) - /// bonus points for making it harder to forget switching on arroy on hannoy when modifying the code let dimensions = self - .readers(wtxn, self.quantized_db()) + ._hannoy_readers(wtxn, self._hannoy_quantized_db()) .next() .transpose()? .map(|reader| reader.dimensions()); @@ -230,7 +159,7 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimensions); + let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimensions); let mut builder = writer.builder(&mut rng); let builder = builder.progress(|step| progress.update_progress_from_arroy(step)); @@ -241,7 +170,7 @@ impl VectorStore { Ok(()) } else { let dimensions = self - .readers(wtxn, self.angular_db()) + ._hannoy_readers(wtxn, self._hannoy_angular_db()) .next() .transpose()? .map(|reader| reader.dimensions()); @@ -250,7 +179,7 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimensions); + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimensions); let mut builder = writer.builder(&mut rng); let builder = builder.progress(|step| progress.update_progress_from_arroy(step)); @@ -271,16 +200,61 @@ impl VectorStore { rng: &mut R, dimension: usize, quantizing: bool, - hannoy_memory: Option, + available_memory: Option, cancel: &(impl Fn() -> bool + Sync + Send), - ) -> Result<(), hannoy::Error> { + ) -> Result<(), crate::Error> { for index in vector_store_range_for_embedder(self.embedder_index) { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimension); + if writer.need_build(wtxn)? { + let mut builder = writer.builder(rng); + let builder = + builder.progress(|step| progress.update_progress_from_arroy(step)); + builder + .available_memory(available_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .build(wtxn)?; + } else if writer.is_empty(wtxn)? { + continue; + } + } else { + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimension); + // If we are quantizing the databases, we can't know from meilisearch + // if the db was empty but still contained the wrong metadata, thus we need + // to quantize everything and can't stop early. Since this operation can + // only happens once in the life of an embedder, it's not very performance + // sensitive. + if quantizing && !self.quantized { + let writer = writer + .prepare_changing_distance::( + wtxn, + )?; + let mut builder = writer.builder(rng); + let builder = + builder.progress(|step| progress.update_progress_from_arroy(step)); + builder + .available_memory(available_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .build(wtxn)?; + } else if writer.need_build(wtxn)? { + let mut builder = writer.builder(rng); + let builder = + builder.progress(|step| progress.update_progress_from_arroy(step)); + builder + .available_memory(available_memory.unwrap_or(usize::MAX)) + .cancel(cancel) + .build(wtxn)?; + } else if writer.is_empty(wtxn)? { + continue; + } + } + } else if self.quantized { + let writer = hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension); if writer.need_build(wtxn)? { let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; @@ -288,24 +262,24 @@ impl VectorStore { continue; } } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimension); // If we are quantizing the databases, we can't know from meilisearch // if the db was empty but still contained the wrong metadata, thus we need // to quantize everything and can't stop early. Since this operation can - // only happens once in the life of an embedder, it's not very performances + // only happens once in the life of an embedder, it's not very performance // sensitive. if quantizing && !self.quantized { let writer = writer.prepare_changing_distance::(wtxn)?; let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; } else if writer.need_build(wtxn)? { let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(hannoy_memory.unwrap_or(usize::MAX)) + .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; @@ -326,16 +300,24 @@ impl VectorStore { wtxn: &mut RwTxn, item_id: hannoy::ItemId, embeddings: &Embeddings, - ) -> Result<(), hannoy::Error> { + ) -> Result<(), crate::Error> { let dimension = embeddings.dimension(); for (index, vector) in vector_store_range_for_embedder(self.embedder_index).zip(embeddings.iter()) { - if self.quantized { - hannoy::Writer::new(self.quantized_db(), index, dimension) + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + arroy::Writer::new(self._arroy_quantized_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } else { + arroy::Writer::new(self._arroy_angular_db(), index, dimension) + .add_item(wtxn, item_id, vector)? + } + } else if self.quantized { + hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension) .add_item(wtxn, item_id, vector)? } else { - hannoy::Writer::new(self.angular_db(), index, dimension) + hannoy::Writer::new(self._hannoy_angular_db(), index, dimension) .add_item(wtxn, item_id, vector)? } } @@ -348,31 +330,22 @@ impl VectorStore { wtxn: &mut RwTxn, item_id: hannoy::ItemId, vector: &[f32], - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._add_item(wtxn, self.quantized_db(), item_id, vector) - } else { - self._add_item(wtxn, self.angular_db(), item_id, vector) - } - } - - fn _add_item( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result<(), hannoy::Error> { - let dimension = vector.len(); - - for index in vector_store_range_for_embedder(self.embedder_index) { - let writer = hannoy::Writer::new(db, index, dimension); - if !writer.contains_item(wtxn, item_id)? { - writer.add_item(wtxn, item_id, vector)?; - break; + ) -> Result<(), crate::Error> { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + self._arroy_add_item(wtxn, self._arroy_quantized_db(), item_id, vector) + .map_err(Into::into) + } else { + self._arroy_add_item(wtxn, self._arroy_angular_db(), item_id, vector) + .map_err(Into::into) } + } else if self.quantized { + self._hannoy_add_item(wtxn, self._hannoy_quantized_db(), item_id, vector) + .map_err(Into::into) + } else { + self._hannoy_add_item(wtxn, self._hannoy_angular_db(), item_id, vector) + .map_err(Into::into) } - Ok(()) } /// Add a vector associated with a document in store specified by its id. @@ -384,27 +357,70 @@ impl VectorStore { item_id: hannoy::ItemId, store_id: u8, vector: &[f32], - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._add_item_in_store(wtxn, self.quantized_db(), item_id, store_id, vector) + ) -> Result<(), crate::Error> { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + self._arroy_add_item_in_store( + wtxn, + self._arroy_quantized_db(), + item_id, + store_id, + vector, + ) + .map_err(Into::into) + } else { + self._arroy_add_item_in_store( + wtxn, + self._arroy_angular_db(), + item_id, + store_id, + vector, + ) + .map_err(Into::into) + } + } else if self.quantized { + self._hannoy_add_item_in_store( + wtxn, + self._hannoy_quantized_db(), + item_id, + store_id, + vector, + ) + .map_err(Into::into) } else { - self._add_item_in_store(wtxn, self.angular_db(), item_id, store_id, vector) + self._hannoy_add_item_in_store( + wtxn, + self._hannoy_angular_db(), + item_id, + store_id, + vector, + ) + .map_err(Into::into) } } - fn _add_item_in_store( + /// Delete one item from its value. + pub fn del_item( &self, wtxn: &mut RwTxn, - db: hannoy::Database, item_id: hannoy::ItemId, - store_id: u8, vector: &[f32], - ) -> Result<(), hannoy::Error> { - let dimension = vector.len(); - - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimension); - writer.add_item(wtxn, item_id, vector) + ) -> Result { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + self._arroy_del_item(wtxn, self._arroy_quantized_db(), item_id, vector) + .map_err(Into::into) + } else { + self._arroy_del_item(wtxn, self._arroy_angular_db(), item_id, vector) + .map_err(Into::into) + } + } else if self.quantized { + self._hannoy_del_item(wtxn, self._hannoy_quantized_db(), item_id, vector) + .map_err(Into::into) + } else { + self._hannoy_del_item(wtxn, self._hannoy_angular_db(), item_id, vector) + .map_err(Into::into) + } } /// Delete all embeddings from a specific `item_id` @@ -413,13 +429,21 @@ impl VectorStore { wtxn: &mut RwTxn, dimension: usize, item_id: hannoy::ItemId, - ) -> Result<(), hannoy::Error> { + ) -> Result<(), crate::Error> { for index in vector_store_range_for_embedder(self.embedder_index) { - if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimension); + writer.del_item(wtxn, item_id)?; + } else { + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimension); + writer.del_item(wtxn, item_id)?; + } + } else if self.quantized { + let writer = hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension); writer.del_item(wtxn, item_id)?; } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimension); writer.del_item(wtxn, item_id)?; } } @@ -440,27 +464,48 @@ impl VectorStore { item_id: hannoy::ItemId, store_id: u8, dimensions: usize, - ) -> Result { - if self.quantized { - self._del_item_in_store(wtxn, self.quantized_db(), item_id, store_id, dimensions) + ) -> Result { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + self._arroy_del_item_in_store( + wtxn, + self._arroy_quantized_db(), + item_id, + store_id, + dimensions, + ) + .map_err(Into::into) + } else { + self._arroy_del_item_in_store( + wtxn, + self._arroy_angular_db(), + item_id, + store_id, + dimensions, + ) + .map_err(Into::into) + } + } else if self.quantized { + self._hannoy_del_item_in_store( + wtxn, + self._hannoy_quantized_db(), + item_id, + store_id, + dimensions, + ) + .map_err(Into::into) } else { - self._del_item_in_store(wtxn, self.angular_db(), item_id, store_id, dimensions) + self._hannoy_del_item_in_store( + wtxn, + self._hannoy_angular_db(), + item_id, + store_id, + dimensions, + ) + .map_err(Into::into) } } - fn _del_item_in_store( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - store_id: u8, - dimensions: usize, - ) -> Result { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimensions); - writer.del_item(wtxn, item_id) - } - /// Removes all items from the store specified by its id. /// /// # Warning @@ -471,68 +516,48 @@ impl VectorStore { wtxn: &mut RwTxn, store_id: u8, dimensions: usize, - ) -> Result<(), hannoy::Error> { - if self.quantized { - self._clear_store(wtxn, self.quantized_db(), store_id, dimensions) - } else { - self._clear_store(wtxn, self.angular_db(), store_id, dimensions) - } - } - - fn _clear_store( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - store_id: u8, - dimensions: usize, - ) -> Result<(), hannoy::Error> { - let index = vector_store_for_embedder(self.embedder_index, store_id); - let writer = hannoy::Writer::new(db, index, dimensions); - writer.clear(wtxn) - } - - /// Delete one item from its value. - pub fn del_item( - &self, - wtxn: &mut RwTxn, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result { - if self.quantized { - self._del_item(wtxn, self.quantized_db(), item_id, vector) - } else { - self._del_item(wtxn, self.angular_db(), item_id, vector) - } - } - - fn _del_item( - &self, - wtxn: &mut RwTxn, - db: hannoy::Database, - item_id: hannoy::ItemId, - vector: &[f32], - ) -> Result { - let dimension = vector.len(); - - for index in vector_store_range_for_embedder(self.embedder_index) { - let writer = hannoy::Writer::new(db, index, dimension); - if writer.contains_item(wtxn, item_id)? { - return writer.del_item(wtxn, item_id); - } - } - Ok(false) - } - - pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), hannoy::Error> { - for index in vector_store_range_for_embedder(self.embedder_index) { + ) -> Result<(), crate::Error> { + if self.backend == VectorStoreBackend::Arroy { if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + self._arroy_clear_store(wtxn, self._arroy_quantized_db(), store_id, dimensions) + .map_err(Into::into) + } else { + self._arroy_clear_store(wtxn, self._arroy_angular_db(), store_id, dimensions) + .map_err(Into::into) + } + } else if self.quantized { + self._hannoy_clear_store(wtxn, self._hannoy_quantized_db(), store_id, dimensions) + .map_err(Into::into) + } else { + self._hannoy_clear_store(wtxn, self._hannoy_angular_db(), store_id, dimensions) + .map_err(Into::into) + } + } + + pub fn clear(&self, wtxn: &mut RwTxn, dimension: usize) -> Result<(), crate::Error> { + for index in vector_store_range_for_embedder(self.embedder_index) { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimension); + if writer.is_empty(wtxn)? { + continue; + } + writer.clear(wtxn)?; + } else { + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimension); + if writer.is_empty(wtxn)? { + continue; + } + writer.clear(wtxn)?; + } + } else if self.quantized { + let writer = hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension); if writer.is_empty(wtxn)? { continue; } writer.clear(wtxn)?; } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimension); if writer.is_empty(wtxn)? { continue; } @@ -551,26 +576,26 @@ impl VectorStore { for index in vector_store_range_for_embedder(self.embedder_index) { let contains = if self.backend == VectorStoreBackend::Arroy { if self.quantized { - let writer = arroy::Writer::new(self.arroy_quantized_db(), index, dimension); + let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } writer.contains_item(rtxn, item)? } else { - let writer = arroy::Writer::new(self.arroy_angular_db(), index, dimension); + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } writer.contains_item(rtxn, item)? } } else if self.quantized { - let writer = hannoy::Writer::new(self.quantized_db(), index, dimension); + let writer = hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } writer.contains_item(rtxn, item)? } else { - let writer = hannoy::Writer::new(self.angular_db(), index, dimension); + let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimension); if writer.is_empty(rtxn)? { continue; } @@ -592,18 +617,345 @@ impl VectorStore { ) -> crate::Result> { if self.backend == VectorStoreBackend::Arroy { if self.quantized { - self._arroy_nns_by_item(rtxn, self.arroy_quantized_db(), item, limit, filter) + self._arroy_nns_by_item(rtxn, self._arroy_quantized_db(), item, limit, filter) .map_err(Into::into) } else { - self._arroy_nns_by_item(rtxn, self.arroy_angular_db(), item, limit, filter) + self._arroy_nns_by_item(rtxn, self._arroy_angular_db(), item, limit, filter) .map_err(Into::into) } } else if self.quantized { - self._nns_by_item(rtxn, self.quantized_db(), item, limit, filter).map_err(Into::into) + self._hannoy_nns_by_item(rtxn, self._hannoy_quantized_db(), item, limit, filter) + .map_err(Into::into) } else { - self._nns_by_item(rtxn, self.angular_db(), item, limit, filter).map_err(Into::into) + self._hannoy_nns_by_item(rtxn, self._hannoy_angular_db(), item, limit, filter) + .map_err(Into::into) } } + pub fn nns_by_vector( + &self, + rtxn: &RoTxn, + vector: &[f32], + limit: usize, + filter: Option<&RoaringBitmap>, + ) -> crate::Result> { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + self._arroy_nns_by_vector(rtxn, self._arroy_quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._arroy_nns_by_vector(rtxn, self._arroy_angular_db(), vector, limit, filter) + .map_err(Into::into) + } + } else if self.quantized { + self._hannoy_nns_by_vector(rtxn, self._hannoy_quantized_db(), vector, limit, filter) + .map_err(Into::into) + } else { + self._hannoy_nns_by_vector(rtxn, self._hannoy_angular_db(), vector, limit, filter) + .map_err(Into::into) + } + } + pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { + let mut vectors = Vec::new(); + + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + for reader in self._arroy_readers(rtxn, self._arroy_quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self._arroy_readers(rtxn, self._arroy_angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } + } else if self.quantized { + for reader in self._hannoy_readers(rtxn, self._hannoy_quantized_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } else { + for reader in self._hannoy_readers(rtxn, self._hannoy_angular_db()) { + if let Some(vec) = reader?.item_vector(rtxn, item_id)? { + vectors.push(vec); + } + } + } + + Ok(vectors) + } + + pub fn aggregate_stats( + &self, + rtxn: &RoTxn, + stats: &mut HannoyStats, + ) -> Result<(), crate::Error> { + if self.backend == VectorStoreBackend::Arroy { + if self.quantized { + for reader in self._arroy_readers(rtxn, self._arroy_quantized_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } else { + for reader in self._arroy_readers(rtxn, self._arroy_angular_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } + } else if self.quantized { + for reader in self._hannoy_readers(rtxn, self._hannoy_quantized_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } else { + for reader in self._hannoy_readers(rtxn, self._hannoy_angular_db()) { + let reader = reader?; + let documents = reader.item_ids(); + stats.documents |= documents; + stats.number_of_embeddings += documents.len(); + } + } + + Ok(()) + } + + // private functions + fn _arroy_readers<'a, D: arroy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: arroy::Database, + ) -> impl Iterator, arroy::Error>> + 'a { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match arroy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(arroy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) + } + + fn _hannoy_readers<'a, D: hannoy::Distance>( + &'a self, + rtxn: &'a RoTxn<'a>, + db: hannoy::Database, + ) -> impl Iterator, hannoy::Error>> + 'a { + vector_store_range_for_embedder(self.embedder_index).filter_map(move |index| { + match hannoy::Reader::open(rtxn, index, db) { + Ok(reader) => match reader.is_empty(rtxn) { + Ok(false) => Some(Ok(reader)), + Ok(true) => None, + Err(e) => Some(Err(e)), + }, + Err(hannoy::Error::MissingMetadata(_)) => None, + Err(e) => Some(Err(e)), + } + }) + } + + fn _arroy_items_in_store( + &self, + rtxn: &RoTxn, + db: arroy::Database, + store_id: u8, + with_items: F, + ) -> Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let reader = arroy::Reader::open(rtxn, index, db); + match reader { + Ok(reader) => Ok(with_items(reader.item_ids())), + Err(arroy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(err) => Err(err), + } + } + + fn _hannoy_items_in_store( + &self, + rtxn: &RoTxn, + db: hannoy::Database, + store_id: u8, + with_items: F, + ) -> Result + where + F: FnOnce(&RoaringBitmap) -> O, + { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let reader = hannoy::Reader::open(rtxn, index, db); + match reader { + Ok(reader) => Ok(with_items(reader.item_ids())), + Err(hannoy::Error::MissingMetadata(_)) => Ok(with_items(&RoaringBitmap::new())), + Err(err) => Err(err), + } + } + + fn _arroy_add_item( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + vector: &[f32], + ) -> Result<(), arroy::Error> { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = arroy::Writer::new(db, index, dimension); + if !writer.contains_item(wtxn, item_id)? { + writer.add_item(wtxn, item_id, vector)?; + break; + } + } + Ok(()) + } + + fn _hannoy_add_item( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); + if !writer.contains_item(wtxn, item_id)? { + writer.add_item(wtxn, item_id, vector)?; + break; + } + } + Ok(()) + } + + fn _arroy_add_item_in_store( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + store_id: u8, + vector: &[f32], + ) -> Result<(), arroy::Error> { + let dimension = vector.len(); + + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = arroy::Writer::new(db, index, dimension); + writer.add_item(wtxn, item_id, vector) + } + + fn _hannoy_add_item_in_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + store_id: u8, + vector: &[f32], + ) -> Result<(), hannoy::Error> { + let dimension = vector.len(); + + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimension); + writer.add_item(wtxn, item_id, vector) + } + + fn _arroy_del_item_in_store( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + store_id: u8, + dimensions: usize, + ) -> Result { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = arroy::Writer::new(db, index, dimensions); + writer.del_item(wtxn, item_id) + } + + fn _hannoy_del_item_in_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + store_id: u8, + dimensions: usize, + ) -> Result { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); + writer.del_item(wtxn, item_id) + } + + fn _arroy_clear_store( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + store_id: u8, + dimensions: usize, + ) -> Result<(), arroy::Error> { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = arroy::Writer::new(db, index, dimensions); + writer.clear(wtxn) + } + + fn _hannoy_clear_store( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + store_id: u8, + dimensions: usize, + ) -> Result<(), hannoy::Error> { + let index = vector_store_for_embedder(self.embedder_index, store_id); + let writer = hannoy::Writer::new(db, index, dimensions); + writer.clear(wtxn) + } + + fn _arroy_del_item( + &self, + wtxn: &mut RwTxn, + db: arroy::Database, + item_id: arroy::ItemId, + vector: &[f32], + ) -> Result { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = arroy::Writer::new(db, index, dimension); + if writer.contains_item(wtxn, item_id)? { + return writer.del_item(wtxn, item_id); + } + } + Ok(false) + } + + fn _hannoy_del_item( + &self, + wtxn: &mut RwTxn, + db: hannoy::Database, + item_id: hannoy::ItemId, + vector: &[f32], + ) -> Result { + let dimension = vector.len(); + + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = hannoy::Writer::new(db, index, dimension); + if writer.contains_item(wtxn, item_id)? { + return writer.del_item(wtxn, item_id); + } + } + Ok(false) + } fn _arroy_nns_by_item( &self, @@ -615,7 +967,7 @@ impl VectorStore { ) -> Result, arroy::Error> { let mut results = Vec::new(); - for reader in self.arroy_readers(rtxn, db) { + for reader in self._arroy_readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); if let Some(filter) = filter { @@ -633,7 +985,7 @@ impl VectorStore { Ok(results) } - fn _nns_by_item( + fn _hannoy_nns_by_item( &self, rtxn: &RoTxn, db: hannoy::Database, @@ -643,7 +995,7 @@ impl VectorStore { ) -> Result, hannoy::Error> { let mut results = Vec::new(); - for reader in self.readers(rtxn, db) { + for reader in self._hannoy_readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); searcher.ef_search((limit * 10).max(100)); // TODO find better ef @@ -659,29 +1011,6 @@ impl VectorStore { Ok(results) } - pub fn nns_by_vector( - &self, - rtxn: &RoTxn, - vector: &[f32], - limit: usize, - filter: Option<&RoaringBitmap>, - ) -> crate::Result> { - if self.backend == VectorStoreBackend::Arroy { - if self.quantized { - self._arroy_nns_by_vector(rtxn, self.arroy_quantized_db(), vector, limit, filter) - .map_err(Into::into) - } else { - self._arroy_nns_by_vector(rtxn, self.arroy_angular_db(), vector, limit, filter) - .map_err(Into::into) - } - } else if self.quantized { - self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) - .map_err(Into::into) - } else { - self._nns_by_vector(rtxn, self.angular_db(), vector, limit, filter).map_err(Into::into) - } - } - fn _arroy_nns_by_vector( &self, rtxn: &RoTxn, @@ -692,7 +1021,7 @@ impl VectorStore { ) -> Result, arroy::Error> { let mut results = Vec::new(); - for reader in self.arroy_readers(rtxn, db) { + for reader in self._arroy_readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); if let Some(filter) = filter { @@ -710,7 +1039,7 @@ impl VectorStore { Ok(results) } - fn _nns_by_vector( + fn _hannoy_nns_by_vector( &self, rtxn: &RoTxn, db: hannoy::Database, @@ -720,7 +1049,7 @@ impl VectorStore { ) -> Result, hannoy::Error> { let mut results = Vec::new(); - for reader in self.readers(rtxn, db) { + for reader in self._hannoy_readers(rtxn, db) { let reader = reader?; let mut searcher = reader.nns(limit); searcher.ef_search((limit * 10).max(100)); // TODO find better ef @@ -736,79 +1065,21 @@ impl VectorStore { Ok(results) } - pub fn item_vectors(&self, rtxn: &RoTxn, item_id: u32) -> crate::Result>> { - let mut vectors = Vec::new(); - - if self.backend == VectorStoreBackend::Arroy { - if self.quantized { - for reader in self.arroy_readers(rtxn, self.arroy_quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } else { - for reader in self.arroy_readers(rtxn, self.arroy_angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } - } else if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } else { - for reader in self.readers(rtxn, self.angular_db()) { - if let Some(vec) = reader?.item_vector(rtxn, item_id)? { - vectors.push(vec); - } - } - } - - Ok(vectors) - } - - fn arroy_angular_db(&self) -> arroy::Database { + fn _arroy_angular_db(&self) -> arroy::Database { self.database.remap_types() } - fn arroy_quantized_db(&self) -> arroy::Database { + fn _arroy_quantized_db(&self) -> arroy::Database { self.database.remap_types() } - fn angular_db(&self) -> hannoy::Database { + fn _hannoy_angular_db(&self) -> hannoy::Database { self.database.remap_data_type() } - fn quantized_db(&self) -> hannoy::Database { + fn _hannoy_quantized_db(&self) -> hannoy::Database { self.database.remap_data_type() } - - pub fn aggregate_stats( - &self, - rtxn: &RoTxn, - stats: &mut HannoyStats, - ) -> Result<(), hannoy::Error> { - if self.quantized { - for reader in self.readers(rtxn, self.quantized_db()) { - let reader = reader?; - let documents = reader.item_ids(); - stats.documents |= documents; - stats.number_of_embeddings += documents.len(); - } - } else { - for reader in self.readers(rtxn, self.angular_db()) { - let reader = reader?; - let documents = reader.item_ids(); - stats.documents |= documents; - stats.number_of_embeddings += documents.len(); - } - } - - Ok(()) - } } #[derive(Debug, Default, Clone)] From 381de52fc5bfdb99cc3558d3077e1306de8ab24c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 12:09:18 +0200 Subject: [PATCH 36/62] Add setting to change backend --- crates/milli/src/update/settings.rs | 62 +++++++++++++++++++++++- crates/milli/src/update/test_settings.rs | 2 + 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 5530ae718..6b5dbd359 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -41,6 +41,7 @@ use crate::vector::settings::{ }; use crate::vector::{ Embedder, EmbeddingConfig, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment, + VectorStoreBackend, }; use crate::{ ChannelCongestion, FieldId, FilterableAttributesRule, Index, LocalizedAttributesRule, Result, @@ -199,6 +200,7 @@ pub struct Settings<'a, 't, 'i> { prefix_search: Setting, facet_search: Setting, chat: Setting, + vector_store: Setting, } impl<'a, 't, 'i> Settings<'a, 't, 'i> { @@ -238,6 +240,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { prefix_search: Setting::NotSet, facet_search: Setting::NotSet, chat: Setting::NotSet, + vector_store: Setting::NotSet, indexer_config, } } @@ -476,6 +479,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { self.chat = Setting::Reset; } + pub fn set_vector_store(&mut self, value: VectorStoreBackend) { + self.vector_store = Setting::Set(value); + } + + pub fn reset_vector_store(&mut self) { + self.vector_store = Setting::Reset; + } + #[tracing::instrument( level = "trace" skip(self, progress_callback, should_abort, settings_diff, embedder_stats), @@ -1417,7 +1428,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { } } - pub fn legacy_execute( + fn legacy_execute( mut self, progress_callback: FP, should_abort: FA, @@ -1486,6 +1497,51 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { Ok(()) } + fn execute_vector_backend<'indexer, MSP>( + &mut self, + must_stop_processing: &'indexer MSP, + progress: &'indexer Progress, + ) -> Result<()> + where + MSP: Fn() -> bool + Sync, + { + let new_backend = match self.vector_store { + Setting::Set(new_backend) => { + self.index.put_vector_store(self.wtxn, new_backend)?; + new_backend + } + Setting::Reset => { + self.index.delete_vector_store(self.wtxn)?; + VectorStoreBackend::default() + } + Setting::NotSet => return Ok(()), + }; + let old_backend = self.index.get_vector_store(self.wtxn)?; + + if old_backend == new_backend { + return Ok(()); + } + + let embedding_configs = self.index.embedding_configs(); + for config in embedding_configs.embedding_configs(self.wtxn)? { + if must_stop_processing() { + return Err(crate::InternalError::AbortedIndexation.into()); + } + /// TODO use the embedder name to display progress + let quantized = config.config.quantized(); + let embedder_id = embedding_configs.embedder_id(self.wtxn, &config.name)?.unwrap(); + let mut vector_store = crate::vector::VectorStore::new( + old_backend, + self.index.vector_store, + embedder_id, + quantized, + ); + vector_store.change_backend(self.wtxn, progress.clone(), must_stop_processing)?; + } + + Ok(()) + } + pub fn execute<'indexer, MSP>( mut self, must_stop_processing: &'indexer MSP, @@ -1495,6 +1551,9 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { where MSP: Fn() -> bool + Sync, { + // execute any pending vector store backend change + self.execute_vector_backend(must_stop_processing, progress)?; + // force the old indexer if the environment says so if self.indexer_config.experimental_no_edition_2024_for_settings { return self @@ -1536,6 +1595,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { facet_search: Setting::NotSet, disable_on_numbers: Setting::NotSet, chat: Setting::NotSet, + vector_store: Setting::NotSet, wtxn: _, index: _, indexer_config: _, diff --git a/crates/milli/src/update/test_settings.rs b/crates/milli/src/update/test_settings.rs index 59e8d9ff1..9e4579667 100644 --- a/crates/milli/src/update/test_settings.rs +++ b/crates/milli/src/update/test_settings.rs @@ -898,6 +898,7 @@ fn test_correct_settings_init() { facet_search, disable_on_numbers, chat, + vector_store, } = settings; assert!(matches!(searchable_fields, Setting::NotSet)); assert!(matches!(displayed_fields, Setting::NotSet)); @@ -927,6 +928,7 @@ fn test_correct_settings_init() { assert!(matches!(facet_search, Setting::NotSet)); assert!(matches!(disable_on_numbers, Setting::NotSet)); assert!(matches!(chat, Setting::NotSet)); + assert!(matches!(vector_store, Setting::NotSet)); }) .unwrap(); } From 231f86decf88a1ce738cc118390c2b80f61d893c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 12:10:13 +0200 Subject: [PATCH 37/62] Refer to v1.19 and remove arroy -> hannoy dumpless upgrade --- Cargo.lock | 34 ++++++++--------- Cargo.toml | 2 +- crates/milli/src/update/upgrade/mod.rs | 7 +--- crates/milli/src/update/upgrade/new_hannoy.rs | 37 ------------------- 4 files changed, 20 insertions(+), 60 deletions(-) delete mode 100644 crates/milli/src/update/upgrade/new_hannoy.rs diff --git a/Cargo.lock b/Cargo.lock index 8fec0620c..5ca158a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "bumpalo", @@ -790,7 +790,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "time", @@ -1784,7 +1784,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "big_s", @@ -2016,7 +2016,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.22.0" +version = "1.19.0" dependencies = [ "tempfile", "thiserror 2.0.14", @@ -2038,7 +2038,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.22.0" +version = "1.19.0" dependencies = [ "insta", "levenshtein_automata", @@ -2060,7 +2060,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.22.0" +version = "1.19.0" dependencies = [ "criterion", "serde_json", @@ -2205,7 +2205,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.22.0" +version = "1.19.0" dependencies = [ "arbitrary", "bumpalo", @@ -3028,7 +3028,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "backoff", @@ -3275,7 +3275,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.22.0" +version = "1.19.0" dependencies = [ "criterion", "serde_json", @@ -3775,7 +3775,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.22.0" +version = "1.19.0" dependencies = [ "insta", "md5", @@ -3786,7 +3786,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.22.0" +version = "1.19.0" dependencies = [ "actix-cors", "actix-http", @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.22.0" +version = "1.19.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3902,7 +3902,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.22.0" +version = "1.19.0" dependencies = [ "actix-web", "anyhow", @@ -3937,7 +3937,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "clap", @@ -3971,7 +3971,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.22.0" +version = "1.19.0" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4554,7 +4554,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.22.0" +version = "1.19.0" dependencies = [ "big_s", "serde_json", @@ -7515,7 +7515,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.22.0" +version = "1.19.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index f4a6372a2..836e08dcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.22.0" +version = "1.19.0" authors = [ "Quentin de Quelen ", "Clément Renault ", diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index b7ff45b77..e9f7cbe21 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -1,4 +1,3 @@ -mod new_hannoy; mod v1_12; mod v1_13; mod v1_14; @@ -6,7 +5,6 @@ mod v1_15; mod v1_16; use heed::RwTxn; -use new_hannoy::Latest_V1_18_New_Hannoy; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; @@ -38,7 +36,7 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, &ToTargetNoOp { target: (1, 18, 0) }, - &Latest_V1_18_New_Hannoy {}, + &ToTargetNoOp { target: (1, 19, 0) }, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -68,8 +66,7 @@ const fn start(from: (u32, u32, u32)) -> Option { (1, 15, _) => function_index!(6), (1, 16, _) | (1, 17, _) => function_index!(7), (1, 18, _) => function_index!(8), - (1, 19, _) => function_index!(8), - (1, 22, _) => function_index!(9), + (1, 19, _) => function_index!(9), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/new_hannoy.rs b/crates/milli/src/update/upgrade/new_hannoy.rs deleted file mode 100644 index 097ec7a7e..000000000 --- a/crates/milli/src/update/upgrade/new_hannoy.rs +++ /dev/null @@ -1,37 +0,0 @@ -use heed::RwTxn; - -use super::UpgradeIndex; -use crate::progress::Progress; -use crate::vector::VectorStore; -use crate::{Index, Result}; - -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_18_New_Hannoy(); - -impl UpgradeIndex for Latest_V1_18_New_Hannoy { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _original: (u32, u32, u32), - progress: Progress, - ) -> Result { - let embedding_configs = index.embedding_configs(); - let backend = index.get_vector_store(wtxn)?; - for config in embedding_configs.embedding_configs(wtxn)? { - // TODO use the embedder name to display progress - /// REMOVE THIS FILE, IMPLEMENT CONVERSION AS A SETTING CHANGE - let quantized = config.config.quantized(); - let embedder_id = embedding_configs.embedder_id(wtxn, &config.name)?.unwrap(); - let mut vector_store = - VectorStore::new(backend, index.vector_store, embedder_id, quantized); - vector_store.change_backend(wtxn, progress.clone())?; - } - - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - (1, 22, 0) - } -} From 8933d870312fd6733625477e9ff092d36bd97e0d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 12:10:36 +0200 Subject: [PATCH 38/62] Make backend change cancelable --- crates/milli/src/vector/store.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index c39596da2..a93ae3043 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -104,7 +104,15 @@ impl VectorStore { } } - pub fn change_backend(&mut self, wtxn: &mut RwTxn, progress: Progress) -> crate::Result<()> { + pub fn change_backend( + &mut self, + wtxn: &mut RwTxn, + progress: Progress, + must_stop_processing: MSP, + ) -> crate::Result<()> + where + MSP: Fn() -> bool + Sync + Send, + { if self.backend == VectorStoreBackend::Arroy { self.backend = VectorStoreBackend::Hannoy; if self.quantized { @@ -139,6 +147,7 @@ impl VectorStore { let mut rng = rand::rngs::StdRng::from_entropy(); let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimensions); let mut builder = writer.builder(&mut rng).progress(progress.clone()); + builder.cancel(must_stop_processing); builder.prepare_arroy_conversion(wtxn)?; builder.build::(wtxn)?; } From 00d1006cd9968b5bb2eb4842a0d8671c286d6ddb Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:16:24 +0200 Subject: [PATCH 39/62] add experimental feature --- crates/index-scheduler/src/features.rs | 13 +++++++++++++ crates/meilisearch-types/src/features.rs | 1 + .../meilisearch/src/analytics/segment_analytics.rs | 3 +++ crates/meilisearch/src/routes/features.rs | 14 ++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index 1b01f89de..5646a5d80 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -158,6 +158,19 @@ impl RoFeatures { .into()) } } + + pub fn check_vector_store_setting(&self, disabled_action: &'static str) -> Result<()> { + if self.runtime.vector_store_setting { + Ok(()) + } else { + Err(FeatureNotEnabledError { + disabled_action, + feature: "vector_store_setting", + issue_link: "https://github.com/orgs/meilisearch/discussions/860", + } + .into()) + } + } } impl FeatureData { diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index cf66422b2..9c2a6a135 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -21,6 +21,7 @@ pub struct RuntimeTogglableFeatures { pub composite_embedders: bool, pub chat_completions: bool, pub multimodal: bool, + pub vector_store_setting: bool, } #[derive(Default, Debug, Clone, Copy)] diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index a2a0f0c05..04020a74f 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -205,6 +205,7 @@ struct Infos { experimental_no_snapshot_compaction: bool, experimental_no_edition_2024_for_dumps: bool, experimental_no_edition_2024_for_settings: bool, + experimental_vector_store_setting: bool, gpu_enabled: bool, db_path: bool, import_dump: bool, @@ -307,6 +308,7 @@ impl Infos { composite_embedders, chat_completions, multimodal, + vector_store_setting, } = features; // We're going to override every sensible information. @@ -332,6 +334,7 @@ impl Infos { experimental_embedding_cache_entries, experimental_no_snapshot_compaction, experimental_no_edition_2024_for_dumps, + experimental_vector_store_setting: vector_store_setting, gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(), db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), diff --git a/crates/meilisearch/src/routes/features.rs b/crates/meilisearch/src/routes/features.rs index 1a1f89b2d..3d4219a12 100644 --- a/crates/meilisearch/src/routes/features.rs +++ b/crates/meilisearch/src/routes/features.rs @@ -55,6 +55,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { composite_embedders: Some(false), chat_completions: Some(false), multimodal: Some(false), + vector_store_setting: Some(false), })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( { @@ -103,6 +104,8 @@ pub struct RuntimeTogglableFeatures { pub chat_completions: Option, #[deserr(default)] pub multimodal: Option, + #[deserr(default)] + pub vector_store_setting: Option, } impl From for RuntimeTogglableFeatures { @@ -117,6 +120,7 @@ impl From for RuntimeTogg composite_embedders, chat_completions, multimodal, + vector_store_setting, } = value; Self { @@ -129,6 +133,7 @@ impl From for RuntimeTogg composite_embedders: Some(composite_embedders), chat_completions: Some(chat_completions), multimodal: Some(multimodal), + vector_store_setting: Some(vector_store_setting), } } } @@ -144,6 +149,7 @@ pub struct PatchExperimentalFeatureAnalytics { composite_embedders: bool, chat_completions: bool, multimodal: bool, + vector_store_setting: bool, } impl Aggregate for PatchExperimentalFeatureAnalytics { @@ -162,6 +168,7 @@ impl Aggregate for PatchExperimentalFeatureAnalytics { composite_embedders: new.composite_embedders, chat_completions: new.chat_completions, multimodal: new.multimodal, + vector_store_setting: new.vector_store_setting, }) } @@ -189,6 +196,7 @@ impl Aggregate for PatchExperimentalFeatureAnalytics { composite_embedders: Some(false), chat_completions: Some(false), multimodal: Some(false), + vector_store_setting: Some(false), })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( { @@ -232,6 +240,10 @@ async fn patch_features( .unwrap_or(old_features.composite_embedders), chat_completions: new_features.0.chat_completions.unwrap_or(old_features.chat_completions), multimodal: new_features.0.multimodal.unwrap_or(old_features.multimodal), + vector_store_setting: new_features + .0 + .vector_store_setting + .unwrap_or(old_features.vector_store_setting), }; // explicitly destructure for analytics rather than using the `Serialize` implementation, because @@ -247,6 +259,7 @@ async fn patch_features( composite_embedders, chat_completions, multimodal, + vector_store_setting, } = new_features; analytics.publish( @@ -260,6 +273,7 @@ async fn patch_features( composite_embedders, chat_completions, multimodal, + vector_store_setting, }, &req, ); From 4ccce18d7bd32978203206dfae6c8aad49f899a5 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:36:24 +0200 Subject: [PATCH 40/62] Add settings route --- crates/dump/src/lib.rs | 1 + crates/dump/src/reader/compat/v5_to_v6.rs | 1 + crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch-types/src/settings.rs | 24 ++++++++++++++++++ .../src/routes/indexes/settings.rs | 25 ++++++++++++++++++- .../src/routes/indexes/settings_analytics.rs | 22 ++++++++++++++++ 6 files changed, 73 insertions(+), 1 deletion(-) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index a2b72e0e5..fdbd701be 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -331,6 +331,7 @@ pub(crate) mod test { facet_search: Setting::NotSet, prefix_search: Setting::NotSet, chat: Setting::NotSet, + vector_store: Setting::NotSet, _kind: std::marker::PhantomData, }; settings.check() diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index cdfa6847e..8261746e0 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -421,6 +421,7 @@ impl From> for v6::Settings { facet_search: v6::Setting::NotSet, prefix_search: v6::Setting::NotSet, chat: v6::Setting::NotSet, + vector_store: v6::Setting::NotSet, _kind: std::marker::PhantomData, } } diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index f651b2352..0f1782568 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -396,6 +396,7 @@ InvalidDocumentEditionContext , InvalidRequest , BAD_REQU InvalidDocumentEditionFunctionFilter , InvalidRequest , BAD_REQUEST ; EditDocumentsByFunctionError , InvalidRequest , BAD_REQUEST ; InvalidSettingsIndexChat , InvalidRequest , BAD_REQUEST ; +InvalidSettingsVectorStore , InvalidRequest , BAD_REQUEST ; // Export InvalidExportUrl , InvalidRequest , BAD_REQUEST ; InvalidExportApiKey , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 9e107a5c3..e1d314b8f 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -14,6 +14,7 @@ use milli::proximity::ProximityPrecision; pub use milli::update::ChatSettings; use milli::update::Setting; use milli::vector::db::IndexEmbeddingConfig; +use milli::vector::VectorStoreBackend; use milli::{Criterion, CriterionError, FilterableAttributesRule, Index, DEFAULT_VALUES_PER_FACET}; use serde::{Deserialize, Serialize, Serializer}; use utoipa::ToSchema; @@ -320,6 +321,11 @@ pub struct Settings { #[schema(value_type = Option)] pub chat: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option)] + pub vector_store: Setting, + #[serde(skip)] #[deserr(skip)] pub _kind: PhantomData, @@ -386,6 +392,7 @@ impl Settings { facet_search: Setting::Reset, prefix_search: Setting::Reset, chat: Setting::Reset, + vector_store: Setting::Reset, _kind: PhantomData, } } @@ -413,6 +420,7 @@ impl Settings { facet_search, prefix_search, chat, + vector_store, _kind, } = self; @@ -437,6 +445,7 @@ impl Settings { localized_attributes: localized_attributes_rules, facet_search, prefix_search, + vector_store, chat, _kind: PhantomData, } @@ -489,6 +498,7 @@ impl Settings { facet_search: self.facet_search, prefix_search: self.prefix_search, chat: self.chat, + vector_store: self.vector_store, _kind: PhantomData, } } @@ -569,6 +579,7 @@ impl Settings { facet_search: other.facet_search.or(self.facet_search), prefix_search: other.prefix_search.or(self.prefix_search), chat: other.chat.clone().or(self.chat.clone()), + vector_store: other.vector_store.or(self.vector_store), _kind: PhantomData, } } @@ -608,6 +619,7 @@ pub fn apply_settings_to_builder( facet_search, prefix_search, chat, + vector_store, _kind, } = settings; @@ -825,6 +837,12 @@ pub fn apply_settings_to_builder( Setting::Reset => builder.reset_chat(), Setting::NotSet => (), } + + match vector_store { + Setting::Set(vector_store) => builder.set_vector_store(*vector_store), + Setting::Reset => builder.reset_vector_store(), + Setting::NotSet => (), + } } pub enum SecretPolicy { @@ -922,6 +940,9 @@ pub fn settings( (name, SettingEmbeddingSettings { inner: Setting::Set(config.into()) }) }) .collect(); + + let vector_store = index.get_vector_store(rtxn)?; + let embedders = Setting::Set(embedders); let search_cutoff_ms = index.search_cutoff(rtxn)?; let localized_attributes_rules = index.localized_attributes_rules(rtxn)?; @@ -968,6 +989,7 @@ pub fn settings( facet_search: Setting::Set(facet_search), prefix_search: Setting::Set(prefix_search.unwrap_or_default()), chat: Setting::Set(chat), + vector_store: Setting::Set(vector_store), _kind: PhantomData, }; @@ -1197,6 +1219,7 @@ pub(crate) mod test { facet_search: Setting::NotSet, prefix_search: Setting::NotSet, chat: Setting::NotSet, + vector_store: Setting::NotSet, _kind: PhantomData::, }; @@ -1229,6 +1252,7 @@ pub(crate) mod test { facet_search: Setting::NotSet, prefix_search: Setting::NotSet, chat: Setting::NotSet, + vector_store: Setting::NotSet, _kind: PhantomData::, }; diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 10120ebff..336740162 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -520,6 +520,17 @@ make_setting_routes!( camelcase_attr: "chat", analytics: ChatAnalytics }, + { + route: "/vector-store", + update_verb: patch, + value_type: meilisearch_types::milli::vector::VectorStoreBackend, + err_type: meilisearch_types::deserr::DeserrJsonError< + meilisearch_types::error::deserr_codes::InvalidSettingsVectorStore, + >, + attr: vector_store, + camelcase_attr: "vectorStore", + analytics: VectorStoreAnalytics + }, ); #[utoipa::path( @@ -610,6 +621,7 @@ pub async fn update_all( facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()), prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()), chat: ChatAnalytics::new(new_settings.chat.as_ref().set()), + vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()), }, &req, ); @@ -665,10 +677,17 @@ pub async fn get_all( let index = index_scheduler.index(&index_uid)?; let rtxn = index.read_txn()?; let mut new_settings = settings(&index, &rtxn, SecretPolicy::HideSecrets)?; - if index_scheduler.features().check_chat_completions("showing index `chat` settings").is_err() { + + let features = index_scheduler.features(); + + if features.check_chat_completions("showing index `chat` settings").is_err() { new_settings.chat = Setting::NotSet; } + if features.check_vector_store_setting("showing index `vectorStore` settings").is_err() { + new_settings.vector_store = Setting::NotSet; + } + debug!(returns = ?new_settings, "Get all settings"); Ok(HttpResponse::Ok().json(new_settings)) } @@ -770,5 +789,9 @@ fn validate_settings( features.check_chat_completions("setting `chat` in the index settings")?; } + if let Setting::Set(_) = &settings.vector_store { + features.check_vector_store_setting("setting `vectorStore` in the index settings")?; + } + Ok(settings.validate()?) } diff --git a/crates/meilisearch/src/routes/indexes/settings_analytics.rs b/crates/meilisearch/src/routes/indexes/settings_analytics.rs index 1b8d0e244..cd573099f 100644 --- a/crates/meilisearch/src/routes/indexes/settings_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/settings_analytics.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::locales::{Locale, LocalizedAttributesRuleView}; use meilisearch_types::milli::update::Setting; +use meilisearch_types::milli::vector::VectorStoreBackend; use meilisearch_types::milli::FilterableAttributesRule; use meilisearch_types::settings::{ ChatSettings, FacetingSettings, PaginationSettings, PrefixSearchSettings, @@ -40,6 +41,7 @@ pub struct SettingsAnalytics { pub facet_search: FacetSearchAnalytics, pub prefix_search: PrefixSearchAnalytics, pub chat: ChatAnalytics, + pub vector_store: VectorStoreAnalytics, } impl Aggregate for SettingsAnalytics { @@ -200,6 +202,10 @@ impl Aggregate for SettingsAnalytics { value: new.prefix_search.value.or(self.prefix_search.value), }, chat: ChatAnalytics { set: new.chat.set | self.chat.set }, + vector_store: VectorStoreAnalytics { + set: new.vector_store.set | self.vector_store.set, + value: new.vector_store.value.or(self.vector_store.value), + }, }) } @@ -693,3 +699,19 @@ impl ChatAnalytics { SettingsAnalytics { chat: self, ..Default::default() } } } + +#[derive(Serialize, Default)] +pub struct VectorStoreAnalytics { + pub set: bool, + pub value: Option, +} + +impl VectorStoreAnalytics { + pub fn new(settings: Option<&VectorStoreBackend>) -> Self { + Self { set: settings.is_some(), value: settings.copied() } + } + + pub fn into_settings(self) -> SettingsAnalytics { + SettingsAnalytics { vector_store: self, ..Default::default() } + } +} From c9cc748f42fd3918b1d0f96248fc6d65665cf49d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:37:52 +0200 Subject: [PATCH 41/62] Mark get_vector_store as public --- crates/milli/src/index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index f8210abfa..7485df855 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -469,7 +469,7 @@ impl Index { )?) } - pub(crate) fn get_vector_store(&self, rtxn: &RoTxn<'_>) -> Result { + pub fn get_vector_store(&self, rtxn: &RoTxn<'_>) -> Result { Ok(self .main .remap_types::>() From a8cc66899c15d5337163b237772f5acbb4a779f3 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:38:18 +0200 Subject: [PATCH 42/62] Derive ToSchema for VectorStoreBackend --- crates/milli/src/vector/store.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index a93ae3043..908c73ca5 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -13,7 +13,18 @@ const HANNOY_EF_CONSTRUCTION: usize = 125; const HANNOY_M: usize = 16; const HANNOY_M0: usize = 32; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Default, + Serialize, + Deserialize, + deserr::Deserr, + utoipa::ToSchema, +)] pub enum VectorStoreBackend { #[default] Arroy, From a989f52657c83899025e6dc794fb66788007de93 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:38:39 +0200 Subject: [PATCH 43/62] Fix signature of backend change function --- crates/milli/src/vector/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 908c73ca5..037c0d565 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -119,10 +119,10 @@ impl VectorStore { &mut self, wtxn: &mut RwTxn, progress: Progress, - must_stop_processing: MSP, + must_stop_processing: &MSP, ) -> crate::Result<()> where - MSP: Fn() -> bool + Sync + Send, + MSP: Fn() -> bool + Sync, { if self.backend == VectorStoreBackend::Arroy { self.backend = VectorStoreBackend::Hannoy; From 118c6da64deb783ccbe9aecca176ef09d64d6018 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 16:42:08 +0200 Subject: [PATCH 44/62] Update hannoy to v0.0.5 --- Cargo.lock | 4 ++-- crates/milli/Cargo.toml | 2 +- crates/milli/src/vector/store.rs | 3 --- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ca158a6c..1fd09697f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2613,9 +2613,9 @@ dependencies = [ [[package]] name = "hannoy" -version = "0.0.4" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80496a4713fefbf4ea388b30288afaa23566173510a694d838e0a85a3ada5c0" +checksum = "4b6a412d145918473a8257706599a1088c505047eef9cc6c63c494c95786044f" dependencies = [ "bytemuck", "byteorder", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 94ac1a09f..29a6b86cf 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -88,7 +88,7 @@ rhai = { version = "1.22.2", features = [ "sync", ] } arroy = "0.6.3" -hannoy = "0.0.4" +hannoy = "0.0.5" rand = "0.8.5" tracing = "0.1.41" ureq = { version = "2.12.1", features = ["json"] } diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 037c0d565..5f2e36b1b 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -274,7 +274,6 @@ impl VectorStore { if writer.need_build(wtxn)? { let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; @@ -292,14 +291,12 @@ impl VectorStore { let writer = writer.prepare_changing_distance::(wtxn)?; let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; } else if writer.need_build(wtxn)? { let mut builder = writer.builder(rng).progress(progress.clone()); builder - .available_memory(available_memory.unwrap_or(usize::MAX)) .cancel(cancel) .ef_construction(HANNOY_EF_CONSTRUCTION) .build::(wtxn)?; From bc5100ddddaeec90bb18e48fa1625147d88cee56 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 17:01:01 +0200 Subject: [PATCH 45/62] Update snap --- .../after_registering_settings_task.snap | 2 +- .../settings_update_processed.snap | 2 +- .../Intel to kefir succeeds.snap | 2 +- .../import_vectors/Intel to kefir.snap | 2 +- .../import_vectors/adding Intel succeeds.snap | 2 +- .../import_vectors/after adding Intel.snap | 2 +- ...ter_registering_settings_task_vectors.snap | 2 +- .../settings_update_processed_vectors.snap | 2 +- .../after_adding_the_documents.snap | 2 +- .../after_adding_the_settings.snap | 2 +- .../after_removing_the_documents.snap | 2 +- .../registered_the_document_deletions.snap | 2 +- ...red_the_setting_and_document_addition.snap | 2 +- .../after_processing_everything.snap | 4 ++-- .../register_automatic_upgrade_task.snap | 2 +- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 4 ++-- .../upgrade_task_failed_again.snap | 4 ++-- .../upgrade_task_succeeded.snap | 4 ++-- crates/meilisearch/tests/dumps/mod.rs | 9 ++++++--- crates/meilisearch/tests/features/mod.rs | 20 ++++++++++++------- crates/meilisearch/tests/upgrade/mod.rs | 4 ++-- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- 30 files changed, 52 insertions(+), 43 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap index a52f18079..568635d05 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/settings_update_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/settings_update_processed.snap index b99e15852..c9c8869a1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/settings_update_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/settings_update_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir succeeds.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir succeeds.snap index 12e03a28b..ecfecbd9c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir succeeds.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir succeeds.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir.snap index 2ea2ebb17..64d6d7713 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/adding Intel succeeds.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/adding Intel succeeds.snap index a2a263b6f..d6fd6f2bf 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/adding Intel succeeds.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/adding Intel succeeds.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after adding Intel.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after adding Intel.snap index 29fc6abf4..af0ad504d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after adding Intel.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after adding Intel.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap index ae943bf48..b238b65af 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/settings_update_processed_vectors.snap b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/settings_update_processed_vectors.snap index 9ada7580a..95283cec6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/settings_update_processed_vectors.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/settings_update_processed_vectors.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap index 96d93de51..32d40e3a7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap index 76a77e5c0..6885e9a48 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap index 422bed51f..0685ab6fa 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_document_ids: 1, deleted_documents: Some(1) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }} 3 {uid: 3, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Index `doggos`: Invalid type for filter subexpression: expected: String, Array, found: true.", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: true, deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap index d8996f82c..7c873a31a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_document_ids: 1, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }} 3 {uid: 3, status: enqueued, details: { original_filter: true, deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap index e7b06eb31..bf4753018 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} +0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, vector_store: NotSet, _kind: PhantomData }, is_deletion: false, allow_index_creation: true }} 1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index e3521f78d..90fb618f1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index 677bb9a07..9a3edc8ed 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 0833b0cb8..fdaecb1b8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index eaec8bd70..fc28df258 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index fb1163613..d09780553 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 9dd2a5642..aa687425e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 22, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 19, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.22.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.19.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/dumps/mod.rs b/crates/meilisearch/tests/dumps/mod.rs index f1bac5297..a1db8efcd 100644 --- a/crates/meilisearch/tests/dumps/mod.rs +++ b/crates/meilisearch/tests/dumps/mod.rs @@ -2189,7 +2189,8 @@ async fn import_dump_v6_containing_experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -2316,7 +2317,8 @@ async fn import_dump_v6_containing_batches_and_enqueued_tasks() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -2423,7 +2425,8 @@ async fn generate_and_import_dump_containing_vectors() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); diff --git a/crates/meilisearch/tests/features/mod.rs b/crates/meilisearch/tests/features/mod.rs index ec5838d35..e0f1afb9b 100644 --- a/crates/meilisearch/tests/features/mod.rs +++ b/crates/meilisearch/tests/features/mod.rs @@ -26,7 +26,8 @@ async fn experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -43,7 +44,8 @@ async fn experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -60,7 +62,8 @@ async fn experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -78,7 +81,8 @@ async fn experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -96,7 +100,8 @@ async fn experimental_features() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); } @@ -121,7 +126,8 @@ async fn experimental_feature_metrics() { "getTaskDocumentsRoute": false, "compositeEmbedders": false, "chatCompletions": false, - "multimodal": false + "multimodal": false, + "vectorStoreSetting": false } "###); @@ -168,7 +174,7 @@ async fn errors() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "Unknown field `NotAFeature`: expected one of `metrics`, `logsRoute`, `editDocumentsByFunction`, `containsFilter`, `network`, `getTaskDocumentsRoute`, `compositeEmbedders`, `chatCompletions`, `multimodal`", + "message": "Unknown field `NotAFeature`: expected one of `metrics`, `logsRoute`, `editDocumentsByFunction`, `containsFilter`, `network`, `getTaskDocumentsRoute`, `compositeEmbedders`, `chatCompletions`, `multimodal`, `vectorStoreSetting`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index bf88352c1..e259b7c42 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.22.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.19.0"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.22.1 is higher than the Meilisearch version 1.22.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.19.1 is higher than the Meilisearch version 1.19.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index c239bd89e..8ca0ef6ac 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index c239bd89e..8ca0ef6ac 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index c239bd89e..8ca0ef6ac 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 465890fd0..ffeabf4ca 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 465890fd0..ffeabf4ca 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 465890fd0..ffeabf4ca 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index c6671f9c2..5267a2edd 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 8eac19d6c..1b482a69e 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.22.0" + "upgradeTo": "v1.19.0" }, "error": null, "duration": "[duration]", From 454581dbc9a5862340585708847c715b7c8b9b43 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 17:48:50 +0200 Subject: [PATCH 46/62] Support progress --- crates/milli/src/update/new/steps.rs | 8 ++++++++ crates/milli/src/update/settings.rs | 26 +++++++++++++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/update/new/steps.rs b/crates/milli/src/update/new/steps.rs index eabf9104e..438354a26 100644 --- a/crates/milli/src/update/new/steps.rs +++ b/crates/milli/src/update/new/steps.rs @@ -21,6 +21,14 @@ make_enum_progress! { } } +make_enum_progress! { + pub enum SettingsIndexerStep { + ChangingVectorStore, + UsingStableIndexer, + UsingExperimentalIndexer, + } +} + make_enum_progress! { pub enum PostProcessingFacets { StringsBulk, diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 6b5dbd359..28d105a75 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -26,11 +26,12 @@ use crate::index::{ DEFAULT_MIN_WORD_LEN_TWO_TYPOS, }; use crate::order_by_map::OrderByMap; -use crate::progress::{EmbedderStats, Progress}; +use crate::progress::{EmbedderStats, Progress, VariableNameStep}; use crate::prompt::{default_max_bytes, default_template_text, PromptData}; use crate::proximity::ProximityPrecision; use crate::update::index_documents::IndexDocumentsMethod; use crate::update::new::indexer::reindex; +use crate::update::new::steps::SettingsIndexerStep; use crate::update::{IndexDocuments, UpdateIndexingStep}; use crate::vector::db::{FragmentConfigs, IndexEmbeddingConfig}; use crate::vector::embedder::{openai, rest}; @@ -1522,14 +1523,23 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { return Ok(()); } - let embedding_configs = self.index.embedding_configs(); - for config in embedding_configs.embedding_configs(self.wtxn)? { + let embedders = self.index.embedding_configs(); + let embedding_configs = embedders.embedding_configs(self.wtxn)?; + enum VectorStoreBackendChangeIndex {} + let embedder_count = embedding_configs.len(); + + for (i, config) in embedding_configs.into_iter().enumerate() { if must_stop_processing() { return Err(crate::InternalError::AbortedIndexation.into()); } - /// TODO use the embedder name to display progress + let embedder_name = &config.name; + progress.update_progress(VariableNameStep::::new( + format!("Changing vector store backend for embedder `{embedder_name}`"), + i as u32, + embedder_count as u32, + )); let quantized = config.config.quantized(); - let embedder_id = embedding_configs.embedder_id(self.wtxn, &config.name)?.unwrap(); + let embedder_id = embedders.embedder_id(self.wtxn, &config.name)?.unwrap(); let mut vector_store = crate::vector::VectorStore::new( old_backend, self.index.vector_store, @@ -1551,11 +1561,13 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { where MSP: Fn() -> bool + Sync, { + progress.update_progress(SettingsIndexerStep::ChangingVectorStore); // execute any pending vector store backend change self.execute_vector_backend(must_stop_processing, progress)?; // force the old indexer if the environment says so if self.indexer_config.experimental_no_edition_2024_for_settings { + progress.update_progress(SettingsIndexerStep::UsingStableIndexer); return self .legacy_execute( |indexing_step| tracing::debug!(update = ?indexing_step), @@ -1601,6 +1613,8 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { indexer_config: _, } = &self { + progress.update_progress(SettingsIndexerStep::UsingExperimentalIndexer); + self.index.set_updated_at(self.wtxn, &OffsetDateTime::now_utc())?; let old_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn, None)?; @@ -1639,6 +1653,8 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { Ok(None) } } else { + progress.update_progress(SettingsIndexerStep::UsingStableIndexer); + self.legacy_execute( |indexing_step| tracing::debug!(update = ?indexing_step), must_stop_processing, From c4848e6cc0b86253cd6dca5eb9390b78a7c399ea Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 1 Sep 2025 17:56:20 +0200 Subject: [PATCH 47/62] Set back snapshot to what it was --- crates/meilisearch/tests/vector/binary_quantized.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index ec82290ff..c5a7c9244 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -104,8 +104,8 @@ async fn binary_quantize_before_sending_documents() { "manual": { "embeddings": [ [ - 0.0, - 0.0, + -1.0, + -1.0, 1.0 ] ], @@ -122,7 +122,7 @@ async fn binary_quantize_before_sending_documents() { [ 1.0, 1.0, - 0.0 + -1.0 ] ], "regenerate": false @@ -191,8 +191,8 @@ async fn binary_quantize_after_sending_documents() { "manual": { "embeddings": [ [ - 0.0, - 0.0, + -1.0, + -1.0, 1.0 ] ], @@ -209,7 +209,7 @@ async fn binary_quantize_after_sending_documents() { [ 1.0, 1.0, - 0.0 + -1.0 ] ], "regenerate": false From a25111f32ea2834f3d6ec80b7fdd36ac7dd57cc4 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 2 Sep 2025 14:52:18 +0200 Subject: [PATCH 48/62] get old backend before it mutates --- crates/milli/src/update/settings.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 28d105a75..7b455c42c 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -1506,6 +1506,8 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { where MSP: Fn() -> bool + Sync, { + let old_backend = self.index.get_vector_store(self.wtxn)?; + let new_backend = match self.vector_store { Setting::Set(new_backend) => { self.index.put_vector_store(self.wtxn, new_backend)?; @@ -1517,7 +1519,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { } Setting::NotSet => return Ok(()), }; - let old_backend = self.index.get_vector_store(self.wtxn)?; if old_backend == new_backend { return Ok(()); From 6b6e69b07af75a215747aa5489f13bcf8eb2a711 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 2 Sep 2025 14:52:43 +0200 Subject: [PATCH 49/62] rename Arroy to "stable" and Hannoy to "experimental" in setting values --- crates/milli/src/vector/store.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 5f2e36b1b..6a18fab1c 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -27,7 +27,11 @@ const HANNOY_M0: usize = 32; )] pub enum VectorStoreBackend { #[default] + #[deserr(rename = "stable")] + #[serde(rename = "stable")] Arroy, + #[deserr(rename = "experimental")] + #[serde(rename = "experimental")] Hannoy, } From 687260bc138365bae241e280809de5c97c93b142 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 2 Sep 2025 17:49:22 +0200 Subject: [PATCH 50/62] Change approach to arroy <-> migration after encountering multiple issues --- crates/milli/src/update/settings.rs | 13 +- crates/milli/src/vector/store.rs | 218 ++++++++++++++++++---------- 2 files changed, 150 insertions(+), 81 deletions(-) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 7b455c42c..bd6e6c301 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -1529,6 +1529,8 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { enum VectorStoreBackendChangeIndex {} let embedder_count = embedding_configs.len(); + let rtxn = self.index.read_txn()?; + for (i, config) in embedding_configs.into_iter().enumerate() { if must_stop_processing() { return Err(crate::InternalError::AbortedIndexation.into()); @@ -1541,13 +1543,20 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { )); let quantized = config.config.quantized(); let embedder_id = embedders.embedder_id(self.wtxn, &config.name)?.unwrap(); - let mut vector_store = crate::vector::VectorStore::new( + let vector_store = crate::vector::VectorStore::new( old_backend, self.index.vector_store, embedder_id, quantized, ); - vector_store.change_backend(self.wtxn, progress.clone(), must_stop_processing)?; + + vector_store.change_backend( + &rtxn, + self.wtxn, + progress.clone(), + must_stop_processing, + self.indexer_config.max_memory, + )?; } Ok(()) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 6a18fab1c..d9caeac03 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -120,35 +120,20 @@ impl VectorStore { } pub fn change_backend( - &mut self, + self, + rtxn: &RoTxn, wtxn: &mut RwTxn, progress: Progress, must_stop_processing: &MSP, + available_memory: Option, ) -> crate::Result<()> where MSP: Fn() -> bool + Sync, { + let mut rng = rand::rngs::StdRng::from_entropy(); if self.backend == VectorStoreBackend::Arroy { - self.backend = VectorStoreBackend::Hannoy; if self.quantized { - let dimensions = self - ._arroy_readers(wtxn, self._arroy_quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); - - let Some(dimensions) = dimensions else { return Ok(()) }; - - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = - hannoy::Writer::new(self._hannoy_quantized_db(), index, dimensions); - let mut builder = writer.builder(&mut rng).progress(progress.clone()); - builder.prepare_arroy_conversion(wtxn)?; - builder.build::(wtxn)?; - } - - Ok(()) + self._arroy_to_hannoy_bq::(rtxn, wtxn, &progress, &mut rng, &must_stop_processing) } else { let dimensions = self ._arroy_readers(wtxn, self._arroy_angular_db()) @@ -159,7 +144,6 @@ impl VectorStore { let Some(dimensions) = dimensions else { return Ok(()) }; for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); let writer = hannoy::Writer::new(self._hannoy_angular_db(), index, dimensions); let mut builder = writer.builder(&mut rng).progress(progress.clone()); builder.cancel(must_stop_processing); @@ -170,28 +154,11 @@ impl VectorStore { Ok(()) } } else { - self.backend = VectorStoreBackend::Arroy; - if self.quantized { - let dimensions = self - ._hannoy_readers(wtxn, self._hannoy_quantized_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); - - let Some(dimensions) = dimensions else { return Ok(()) }; - - for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); - let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimensions); - let mut builder = writer.builder(&mut rng); - let builder = - builder.progress(|step| progress.update_progress_from_arroy(step)); - builder.prepare_hannoy_conversion(wtxn)?; - builder.build(wtxn)?; - } - - Ok(()) + self._hannoy_to_arroy_bq::< + hannoy::distances::Hamming, + arroy::distances::BinaryQuantizedCosine, + _>(rtxn, wtxn, &progress, &mut rng, available_memory, &must_stop_processing) } else { let dimensions = self ._hannoy_readers(wtxn, self._hannoy_angular_db()) @@ -202,7 +169,6 @@ impl VectorStore { let Some(dimensions) = dimensions else { return Ok(()) }; for index in vector_store_range_for_embedder(self.embedder_index) { - let mut rng = rand::rngs::StdRng::from_entropy(); let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimensions); let mut builder = writer.builder(&mut rng); let builder = @@ -232,13 +198,7 @@ impl VectorStore { if self.quantized { let writer = arroy::Writer::new(self._arroy_quantized_db(), index, dimension); if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng); - let builder = - builder.progress(|step| progress.update_progress_from_arroy(step)); - builder - .available_memory(available_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .build(wtxn)?; + arroy_build(wtxn, &progress, rng, available_memory, cancel, &writer)?; } else if writer.is_empty(wtxn)? { continue; } @@ -254,21 +214,9 @@ impl VectorStore { .prepare_changing_distance::( wtxn, )?; - let mut builder = writer.builder(rng); - let builder = - builder.progress(|step| progress.update_progress_from_arroy(step)); - builder - .available_memory(available_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .build(wtxn)?; + arroy_build(wtxn, &progress, rng, available_memory, cancel, &writer)?; } else if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng); - let builder = - builder.progress(|step| progress.update_progress_from_arroy(step)); - builder - .available_memory(available_memory.unwrap_or(usize::MAX)) - .cancel(cancel) - .build(wtxn)?; + arroy_build(wtxn, &progress, rng, available_memory, cancel, &writer)?; } else if writer.is_empty(wtxn)? { continue; } @@ -276,11 +224,7 @@ impl VectorStore { } else if self.quantized { let writer = hannoy::Writer::new(self._hannoy_quantized_db(), index, dimension); if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; + hannoy_build(wtxn, &progress, rng, cancel, &writer)?; } else if writer.is_empty(wtxn)? { continue; } @@ -293,17 +237,9 @@ impl VectorStore { // sensitive. if quantizing && !self.quantized { let writer = writer.prepare_changing_distance::(wtxn)?; - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; + hannoy_build(wtxn, &progress, rng, cancel, &writer)?; } else if writer.need_build(wtxn)? { - let mut builder = writer.builder(rng).progress(progress.clone()); - builder - .cancel(cancel) - .ef_construction(HANNOY_EF_CONSTRUCTION) - .build::(wtxn)?; + hannoy_build(wtxn, &progress, rng, cancel, &writer)?; } else if writer.is_empty(wtxn)? { continue; } @@ -1101,6 +1037,130 @@ impl VectorStore { fn _hannoy_quantized_db(&self) -> hannoy::Database { self.database.remap_data_type() } + + fn _arroy_to_hannoy_bq( + self, + arroy_rtxn: &RoTxn, + hannoy_wtxn: &mut RwTxn, + progress: &Progress, + rng: &mut R, + cancel: &(impl Fn() -> bool + Sync + Send), + ) -> crate::Result<()> + where + R: rand::Rng + rand::SeedableRng, + { + for index in vector_store_range_for_embedder(self.embedder_index) { + let arroy_reader: arroy::Reader = + match arroy::Reader::open(arroy_rtxn, index, self.database.remap_types()) { + Ok(reader) => reader, + Err(arroy::Error::MissingMetadata(_)) => continue, + Err(err) => return Err(err.into()), + }; + let dimensions = arroy_reader.dimensions(); + let hannoy_writer: hannoy::Writer = + hannoy::Writer::new(self.database.remap_types(), index, dimensions); + hannoy_writer.clear(hannoy_wtxn)?; + for entry in arroy_reader.iter(arroy_rtxn)? { + let (item, mut vector) = entry?; + // arroy bug? the `vector` here can be longer than `dimensions`. + // workaround: truncating. + if vector.len() > dimensions { + vector.truncate(dimensions); + } + hannoy_writer.add_item(hannoy_wtxn, item, &vector)?; + } + hannoy_build(hannoy_wtxn, progress, rng, cancel, &hannoy_writer)?; + } + Ok(()) + } + + fn _hannoy_to_arroy_bq( + self, + hannoy_rtxn: &RoTxn, + arroy_wtxn: &mut RwTxn, + progress: &Progress, + rng: &mut R, + available_memory: Option, + cancel: &(impl Fn() -> bool + Sync + Send), + ) -> crate::Result<()> + where + R: rand::Rng + rand::SeedableRng, + { + for index in vector_store_range_for_embedder(self.embedder_index) { + let hannoy_reader: hannoy::Reader = + match hannoy::Reader::open(hannoy_rtxn, index, self.database.remap_types()) { + Ok(reader) => reader, + Err(hannoy::Error::MissingMetadata(_)) => continue, + Err(err) => return Err(err.into()), + }; + let dimensions = hannoy_reader.dimensions(); + let arroy_writer: arroy::Writer = + arroy::Writer::new(self.database.remap_types(), index, dimensions); + arroy_writer.clear(arroy_wtxn)?; + for entry in hannoy_reader.iter(hannoy_rtxn)? { + let (item, mut vector) = entry?; + // hannoy bug? the `vector` here can be longer than `dimensions`. + // workaround: truncating. + if vector.len() > dimensions { + vector.truncate(dimensions); + } + // arroy and hannoy disagreement over the 0 value + // - arroy does: + // - if x >= 0 => 1 + // - if x < 0 => -1 + // - hannoy does: + // - if x > 0 => 1 + // - if x <= 0 => 0 + // because of this, a 0 from a bq hannoy will be converted to a 1 in arroy, destroying the information. + // to fix that, we subtract 0.5 from the hannoy vector, so that any zero value is translated to a strictly + // negative value. + for x in &mut vector { + *x -= 0.5; + } + + arroy_writer.add_item(arroy_wtxn, item, &vector)?; + } + arroy_build(arroy_wtxn, progress, rng, available_memory, cancel, &arroy_writer)?; + } + Ok(()) + } +} + +fn arroy_build( + wtxn: &mut RwTxn<'_>, + progress: &Progress, + rng: &mut R, + available_memory: Option, + cancel: &(impl Fn() -> bool + Sync + Send), + writer: &arroy::Writer, +) -> Result<(), crate::Error> +where + R: rand::Rng + rand::SeedableRng, + D: arroy::Distance, +{ + let mut builder = writer.builder(rng); + let builder = builder.progress(|step| progress.update_progress_from_arroy(step)); + builder.available_memory(available_memory.unwrap_or(usize::MAX)).cancel(cancel).build(wtxn)?; + Ok(()) +} + +fn hannoy_build( + wtxn: &mut RwTxn<'_>, + progress: &Progress, + rng: &mut R, + cancel: &(impl Fn() -> bool + Sync + Send), + writer: &hannoy::Writer, +) -> Result<(), crate::Error> +where + R: rand::Rng + rand::SeedableRng, + D: hannoy::Distance, +{ + let mut builder = writer.builder(rng).progress(progress.clone()); + builder + .cancel(cancel) + .ef_construction(HANNOY_EF_CONSTRUCTION) + .build::(wtxn)?; + Ok(()) } #[derive(Debug, Default, Clone)] From 34eba61c0d46ce232b3f7daf33c4d123dc492696 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 13:42:56 +0200 Subject: [PATCH 51/62] Add new tests --- .../tests/settings/get_settings.rs | 5 + .../tests/vector/binary_quantized.rs | 188 ++++++++++++++++++ 2 files changed, 193 insertions(+) diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index 8419f640d..d9bb78d6c 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -194,6 +194,11 @@ test_setting_routes!( "searchParameters": {} } }, + { + setting: vector_store, + update_verb: patch, + default_value: "stable" + }, ); #[actix_rt::test] diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index c5a7c9244..effa98b4e 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -335,3 +335,191 @@ async fn binary_quantize_clear_documents() { } "#); } + +#[actix_rt::test] +async fn binary_quantize_change_backend() { + let server = Server::new().await; + let index = server.unique_index(); + server.set_features(json!({"vectorStoreSetting": true})).await; + let (response, code) = index + .update_settings(json!({ + "vectorStore": "stable" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (response, code) = index + .update_settings(json!({ + "embedders": { + "manual": { + "source": "userProvided", + "dimensions": 3, + "binaryQuantized": true, + } + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let documents = json!([ + {"id": 0, "name": "kefir", "_vectors": { "manual": [-1.2, -2.3, 3.2] }}, + {"id": 1, "name": "echo", "_vectors": { "manual": [2.5, 1.5, -130] }}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + server.wait_task(value.uid()).await.succeeded(); + + // Make sure the documents are binary quantized + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(json_string!(documents), @r###" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "manual": { + "embeddings": [ + [ + -1.0, + -1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "manual": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": false + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 2 + } + "###); + + let (response, code) = index + .update_settings(json!({ + "vectorStore": "experimental" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(json_string!(documents), @r###" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "manual": { + "embeddings": [ + [ + 0.0, + 0.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "manual": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": false + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 2 + } + "###); + + let (response, code) = index + .update_settings(json!({ + "vectorStore": "stable" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(json_string!(documents), @r###" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "manual": { + "embeddings": [ + [ + -1.0, + -1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "manual": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": false + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 2 + } + "###); +} From 90cc5263f6a2cd939eef7dbb1d2a69794881cf21 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 13:57:58 +0200 Subject: [PATCH 52/62] Remove MEILI_EMBEDDINGS_CHUNK_SIZE --- crates/milli/src/vector/embedder/rest.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/milli/src/vector/embedder/rest.rs b/crates/milli/src/vector/embedder/rest.rs index 3e0c5989a..7c0213c76 100644 --- a/crates/milli/src/vector/embedder/rest.rs +++ b/crates/milli/src/vector/embedder/rest.rs @@ -319,14 +319,7 @@ impl Embedder { pub fn prompt_count_in_chunk_hint(&self) -> usize { match self.data.request.input_type() { InputType::Text => 1, - InputType::TextArray => { - let chunk_size = std::env::var("MEILI_EMBEDDINGS_CHUNK_SIZE") - .ok() - .and_then(|chunk_size| chunk_size.parse().ok()) - .unwrap_or(10); - assert!(chunk_size <= 100, "Embedding chunk size cannot exceed 100"); - chunk_size - } + InputType::TextArray => 10, } } From b05bcf2c13a5211464b2ad69758197cf9722df18 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 14:13:08 +0200 Subject: [PATCH 53/62] happy clippy --- crates/milli/src/vector/store.rs | 37 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index d9caeac03..e15c690a6 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -153,32 +153,29 @@ impl VectorStore { Ok(()) } - } else { - if self.quantized { - self._hannoy_to_arroy_bq::< + } else if self.quantized { + self._hannoy_to_arroy_bq::< hannoy::distances::Hamming, arroy::distances::BinaryQuantizedCosine, _>(rtxn, wtxn, &progress, &mut rng, available_memory, &must_stop_processing) - } else { - let dimensions = self - ._hannoy_readers(wtxn, self._hannoy_angular_db()) - .next() - .transpose()? - .map(|reader| reader.dimensions()); + } else { + let dimensions = self + ._hannoy_readers(wtxn, self._hannoy_angular_db()) + .next() + .transpose()? + .map(|reader| reader.dimensions()); - let Some(dimensions) = dimensions else { return Ok(()) }; + let Some(dimensions) = dimensions else { return Ok(()) }; - for index in vector_store_range_for_embedder(self.embedder_index) { - let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimensions); - let mut builder = writer.builder(&mut rng); - let builder = - builder.progress(|step| progress.update_progress_from_arroy(step)); - builder.prepare_hannoy_conversion(wtxn)?; - builder.build(wtxn)?; - } - - Ok(()) + for index in vector_store_range_for_embedder(self.embedder_index) { + let writer = arroy::Writer::new(self._arroy_angular_db(), index, dimensions); + let mut builder = writer.builder(&mut rng); + let builder = builder.progress(|step| progress.update_progress_from_arroy(step)); + builder.prepare_hannoy_conversion(wtxn)?; + builder.build(wtxn)?; } + + Ok(()) } } From c32c74671d1f8d39d63b1ca380422ecedb80ab60 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 14:45:31 +0200 Subject: [PATCH 54/62] Rename HannoyStats to VectorStoreStats The stats can be provided by any backend --- crates/index-scheduler/src/index_mapper/mod.rs | 6 +++--- crates/milli/src/index.rs | 6 +++--- crates/milli/src/search/facet/filter_vector.rs | 8 ++++---- crates/milli/src/vector/mod.rs | 2 +- crates/milli/src/vector/store.rs | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index 7014f87cb..6103fe7fc 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -143,10 +143,10 @@ impl IndexStats { /// /// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`. pub fn new(index: &Index, rtxn: &RoTxn) -> milli::Result { - let hannoy_stats = index.hannoy_stats(rtxn)?; + let vector_store_stats = index.vector_store_stats(rtxn)?; Ok(IndexStats { - number_of_embeddings: Some(hannoy_stats.number_of_embeddings), - number_of_embedded_documents: Some(hannoy_stats.documents.len()), + number_of_embeddings: Some(vector_store_stats.number_of_embeddings), + number_of_embedded_documents: Some(vector_store_stats.documents.len()), documents_database_stats: index.documents_stats(rtxn)?.unwrap_or_default(), number_of_documents: None, database_size: index.on_disk_size()?, diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 7485df855..0f4f7c455 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -31,7 +31,7 @@ use crate::prompt::PromptData; use crate::proximity::ProximityPrecision; use crate::update::new::StdResult; use crate::vector::db::IndexEmbeddingConfigs; -use crate::vector::{Embedding, HannoyStats, VectorStore, VectorStoreBackend}; +use crate::vector::{Embedding, VectorStore, VectorStoreBackend, VectorStoreStats}; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldIdWordCountCodec, @@ -1825,8 +1825,8 @@ impl Index { Ok(PrefixSettings { compute_prefixes, max_prefix_length: 4, prefix_count_threshold: 100 }) } - pub fn hannoy_stats(&self, rtxn: &RoTxn<'_>) -> Result { - let mut stats = HannoyStats::default(); + pub fn vector_store_stats(&self, rtxn: &RoTxn<'_>) -> Result { + let mut stats = VectorStoreStats::default(); let embedding_configs = self.embedding_configs(); let backend = self.get_vector_store(rtxn)?; diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index d8e007760..04dbc1eef 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -3,7 +3,7 @@ use roaring::{MultiOps, RoaringBitmap}; use crate::error::{DidYouMean, Error}; use crate::vector::db::IndexEmbeddingConfig; -use crate::vector::{HannoyStats, VectorStore}; +use crate::vector::{VectorStoreStats, VectorStore}; use crate::Index; #[derive(Debug, thiserror::Error)] @@ -134,7 +134,7 @@ fn evaluate_inner( } let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = HannoyStats::default(); + let mut stats = VectorStoreStats::default(); vector_store.aggregate_stats(rtxn, &mut stats)?; stats.documents - user_provided_docids.clone() } @@ -143,13 +143,13 @@ fn evaluate_inner( user_provided_docids.clone() } VectorFilter::Regenerate => { - let mut stats = HannoyStats::default(); + let mut stats = VectorStoreStats::default(); vector_store.aggregate_stats(rtxn, &mut stats)?; let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); stats.documents - skip_regenerate } VectorFilter::None => { - let mut stats = HannoyStats::default(); + let mut stats = VectorStoreStats::default(); vector_store.aggregate_stats(rtxn, &mut stats)?; stats.documents } diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index be747f459..fa8b2dba0 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -19,7 +19,7 @@ pub use distribution::DistributionShift; pub use embedder::{Embedder, EmbedderOptions, EmbeddingConfig, SearchQuery}; pub use embeddings::Embeddings; pub use runtime::{RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment}; -pub use store::{HannoyStats, VectorStore, VectorStoreBackend}; +pub use store::{VectorStore, VectorStoreBackend, VectorStoreStats}; pub const REQUEST_PARALLELISM: usize = 40; diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index e15c690a6..11385a0e4 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -645,7 +645,7 @@ impl VectorStore { pub fn aggregate_stats( &self, rtxn: &RoTxn, - stats: &mut HannoyStats, + stats: &mut VectorStoreStats, ) -> Result<(), crate::Error> { if self.backend == VectorStoreBackend::Arroy { if self.quantized { @@ -1161,7 +1161,7 @@ where } #[derive(Debug, Default, Clone)] -pub struct HannoyStats { +pub struct VectorStoreStats { pub number_of_embeddings: u64, pub documents: RoaringBitmap, } From 0faf4951733583c64447114f0d49d82a5381f8b6 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 14:49:24 +0200 Subject: [PATCH 55/62] cargo fmt --- crates/milli/src/search/facet/filter_vector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 04dbc1eef..2e34eb068 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -3,7 +3,7 @@ use roaring::{MultiOps, RoaringBitmap}; use crate::error::{DidYouMean, Error}; use crate::vector::db::IndexEmbeddingConfig; -use crate::vector::{VectorStoreStats, VectorStore}; +use crate::vector::{VectorStore, VectorStoreStats}; use crate::Index; #[derive(Debug, thiserror::Error)] From 13df9645641846bc76bcc6d6779606f47d7bfe26 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Sep 2025 15:08:40 +0200 Subject: [PATCH 56/62] Adopt neutral terminology where arroy/hannoy would be confusing --- .../tests/vector/binary_quantized.rs | 2 +- crates/meilisearch/tests/vector/mod.rs | 4 +- crates/meilisearch/tests/vector/settings.rs | 2 +- crates/meilitool/src/main.rs | 2 +- crates/milli/src/index.rs | 6 +- .../milli/src/update/index_documents/mod.rs | 4 +- .../src/update/index_documents/transform.rs | 10 +-- crates/milli/src/update/new/channel.rs | 63 +++++++++---------- crates/milli/src/update/new/indexer/mod.rs | 24 +++---- crates/milli/src/update/new/indexer/write.rs | 36 +++++------ 10 files changed, 75 insertions(+), 78 deletions(-) diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index effa98b4e..ff3fc470e 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -320,7 +320,7 @@ async fn binary_quantize_clear_documents() { } "###); - // Make sure the hannoy DB has been cleared + // Make sure the vector DB has been cleared let (documents, _code) = index.search_post(json!({ "hybrid": { "embedder": "manual" }, "vector": [1, 1, 1] })).await; snapshot!(documents, @r#" diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 9eb7deff6..d89fbf0a9 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -10,11 +10,11 @@ use std::str::FromStr; use meili_snap::{json_string, snapshot}; use meilisearch::option::MaxThreads; +pub use rest::create_mock; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; -pub use rest::create_mock; pub async fn get_server_vector() -> Server { Server::new().await @@ -684,7 +684,7 @@ async fn clear_documents() { } "###); - // Make sure the hannoy DB has been cleared + // Make sure the vector DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "manual"} })).await; snapshot!(documents, @r#" diff --git a/crates/meilisearch/tests/vector/settings.rs b/crates/meilisearch/tests/vector/settings.rs index 8ace8f092..292451ec3 100644 --- a/crates/meilisearch/tests/vector/settings.rs +++ b/crates/meilisearch/tests/vector/settings.rs @@ -236,7 +236,7 @@ async fn reset_embedder_documents() { } "###); - // Make sure the hannoy DB has been cleared + // Make sure the vector DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "default"} })).await; snapshot!(json_string!(documents), @r###" diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 1a2110b5d..831bcf209 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -142,7 +142,7 @@ enum Command { #[derive(Clone, ValueEnum)] enum IndexPart { - /// Will make the hannoy index hot. + /// Will make the vector index hot. Hannoy, } diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 0f4f7c455..54086d8a5 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -178,7 +178,7 @@ pub struct Index { /// Maps the document id, the facet field id and the strings. pub field_id_docid_facet_strings: Database, - /// Maps an embedder name to its id in the hannoy store. + /// Maps an embedder name to its id in the vector store. pub(crate) embedder_category_id: Database, /// Vector store based on hannoy™. pub vector_store: hannoy::Database, @@ -1881,7 +1881,7 @@ impl Index { facet_id_is_empty_docids, field_id_docid_facet_f64s, field_id_docid_facet_strings, - vector_store: vector_hannoy, + vector_store, embedder_category_id, documents, } = self; @@ -1952,7 +1952,7 @@ impl Index { "field_id_docid_facet_strings", field_id_docid_facet_strings.stat(rtxn).map(compute_size)?, ); - sizes.insert("vector_hannoy", vector_hannoy.stat(rtxn).map(compute_size)?); + sizes.insert("vector_store", vector_store.stat(rtxn).map(compute_size)?); sizes.insert("embedder_category_id", embedder_category_id.stat(rtxn).map(compute_size)?); sizes.insert("documents", documents.stat(rtxn).map(compute_size)?); diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 686da865c..ea468f08b 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -505,7 +505,7 @@ where for (embedder_name, dimension) in dimension { let wtxn = &mut *self.wtxn; - let vector_hannoy = self.index.vector_store; + let vector_store = self.index.vector_store; let cancel = &self.should_abort; let embedder_index = @@ -525,7 +525,7 @@ where pool.install(|| { let mut writer = - VectorStore::new(backend, vector_hannoy, embedder_index, was_quantized); + VectorStore::new(backend, vector_store, embedder_index, was_quantized); writer.build_and_quantize( wtxn, // In the settings we don't have any progress to share diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index e4bd49434..8df8722c3 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -948,13 +948,13 @@ impl<'a, 'i> Transform<'a, 'i> { else { continue; }; - let hannoy = VectorStore::new( + let vector_store = VectorStore::new( backend, self.index.vector_store, infos.embedder_id, was_quantized, ); - let Some(dimensions) = hannoy.dimensions(wtxn)? else { + let Some(dimensions) = vector_store.dimensions(wtxn)? else { continue; }; for fragment_id in fragment_ids { @@ -962,17 +962,17 @@ impl<'a, 'i> Transform<'a, 'i> { if infos.embedding_status.user_provided_docids().is_empty() { // no user provided: clear store - hannoy.clear_store(wtxn, *fragment_id, dimensions)?; + vector_store.clear_store(wtxn, *fragment_id, dimensions)?; continue; } // some user provided, remove only the ids that are not user provided - let to_delete = hannoy.items_in_store(wtxn, *fragment_id, |items| { + let to_delete = vector_store.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() })?; for to_delete in to_delete { - hannoy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; + vector_store.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; } } } diff --git a/crates/milli/src/update/new/channel.rs b/crates/milli/src/update/new/channel.rs index 86843795b..884f133d6 100644 --- a/crates/milli/src/update/new/channel.rs +++ b/crates/milli/src/update/new/channel.rs @@ -255,9 +255,9 @@ impl<'a> From> for FrameWithHeader<'a> { #[repr(u8)] pub enum EntryHeader { DbOperation(DbOperation), - HannoyDeleteVector(HannoyDeleteVector), - HannoySetVectors(HannoySetVectors), - HannoySetVector(HannoySetVector), + DeleteVector(DeleteVector), + SetVectors(SetVectors), + SetVector(SetVector), } impl EntryHeader { @@ -268,9 +268,9 @@ impl EntryHeader { const fn variant_id(&self) -> u8 { match self { EntryHeader::DbOperation(_) => 0, - EntryHeader::HannoyDeleteVector(_) => 1, - EntryHeader::HannoySetVectors(_) => 2, - EntryHeader::HannoySetVector(_) => 3, + EntryHeader::DeleteVector(_) => 1, + EntryHeader::SetVectors(_) => 2, + EntryHeader::SetVector(_) => 3, } } @@ -286,26 +286,26 @@ impl EntryHeader { } const fn total_delete_vector_size() -> usize { - Self::variant_size() + mem::size_of::() + Self::variant_size() + mem::size_of::() } /// The `dimensions` corresponds to the number of `f32` in the embedding. fn total_set_vectors_size(count: usize, dimensions: usize) -> usize { let embedding_size = dimensions * mem::size_of::(); - Self::variant_size() + mem::size_of::() + embedding_size * count + Self::variant_size() + mem::size_of::() + embedding_size * count } fn total_set_vector_size(dimensions: usize) -> usize { let embedding_size = dimensions * mem::size_of::(); - Self::variant_size() + mem::size_of::() + embedding_size + Self::variant_size() + mem::size_of::() + embedding_size } fn header_size(&self) -> usize { let payload_size = match self { EntryHeader::DbOperation(op) => mem::size_of_val(op), - EntryHeader::HannoyDeleteVector(adv) => mem::size_of_val(adv), - EntryHeader::HannoySetVectors(asvs) => mem::size_of_val(asvs), - EntryHeader::HannoySetVector(asv) => mem::size_of_val(asv), + EntryHeader::DeleteVector(adv) => mem::size_of_val(adv), + EntryHeader::SetVectors(asvs) => mem::size_of_val(asvs), + EntryHeader::SetVector(asv) => mem::size_of_val(asv), }; Self::variant_size() + payload_size } @@ -319,19 +319,19 @@ impl EntryHeader { EntryHeader::DbOperation(header) } 1 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::HannoyDeleteVector(header) + EntryHeader::DeleteVector(header) } 2 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::HannoySetVectors(header) + EntryHeader::SetVectors(header) } 3 => { - let header_bytes = &remaining[..mem::size_of::()]; + let header_bytes = &remaining[..mem::size_of::()]; let header = checked::pod_read_unaligned(header_bytes); - EntryHeader::HannoySetVector(header) + EntryHeader::SetVector(header) } id => panic!("invalid variant id: {id}"), } @@ -341,9 +341,9 @@ impl EntryHeader { let (first, remaining) = header_bytes.split_first_mut().unwrap(); let payload_bytes = match self { EntryHeader::DbOperation(op) => bytemuck::bytes_of(op), - EntryHeader::HannoyDeleteVector(adv) => bytemuck::bytes_of(adv), - EntryHeader::HannoySetVectors(asvs) => bytemuck::bytes_of(asvs), - EntryHeader::HannoySetVector(asv) => bytemuck::bytes_of(asv), + EntryHeader::DeleteVector(adv) => bytemuck::bytes_of(adv), + EntryHeader::SetVectors(asvs) => bytemuck::bytes_of(asvs), + EntryHeader::SetVector(asv) => bytemuck::bytes_of(asv), }; *first = self.variant_id(); remaining.copy_from_slice(payload_bytes); @@ -378,7 +378,7 @@ impl DbOperation { #[derive(Debug, Clone, Copy, NoUninit, CheckedBitPattern)] #[repr(transparent)] -pub struct HannoyDeleteVector { +pub struct DeleteVector { pub docid: DocumentId, } @@ -386,13 +386,13 @@ pub struct HannoyDeleteVector { #[repr(C)] /// The embeddings are in the remaining space and represents /// non-aligned [f32] each with dimensions f32s. -pub struct HannoySetVectors { +pub struct SetVectors { pub docid: DocumentId, pub embedder_id: u8, _padding: [u8; 3], } -impl HannoySetVectors { +impl SetVectors { fn embeddings_bytes<'a>(frame: &'a FrameGrantR<'_>) -> &'a [u8] { let skip = EntryHeader::variant_size() + mem::size_of::(); &frame[skip..] @@ -416,14 +416,14 @@ impl HannoySetVectors { #[repr(C)] /// The embeddings are in the remaining space and represents /// non-aligned [f32] each with dimensions f32s. -pub struct HannoySetVector { +pub struct SetVector { pub docid: DocumentId, pub embedder_id: u8, pub extractor_id: u8, _padding: [u8; 2], } -impl HannoySetVector { +impl SetVector { fn embeddings_bytes<'a>(frame: &'a FrameGrantR<'_>) -> &'a [u8] { let skip = EntryHeader::variant_size() + mem::size_of::(); &frame[skip..] @@ -553,7 +553,7 @@ impl<'b> ExtractorBbqueueSender<'b> { let refcell = self.producers.get().unwrap(); let mut producer = refcell.0.borrow_mut_or_yield(); - let payload_header = EntryHeader::HannoyDeleteVector(HannoyDeleteVector { docid }); + let payload_header = EntryHeader::DeleteVector(DeleteVector { docid }); let total_length = EntryHeader::total_delete_vector_size(); if total_length > max_grant { panic!("The entry is larger ({total_length} bytes) than the BBQueue max grant ({max_grant} bytes)"); @@ -589,8 +589,8 @@ impl<'b> ExtractorBbqueueSender<'b> { // to zero to allocate no extra space at all let dimensions = embeddings.first().map_or(0, |emb| emb.len()); - let hannoy_set_vector = HannoySetVectors { docid, embedder_id, _padding: [0; 3] }; - let payload_header = EntryHeader::HannoySetVectors(hannoy_set_vector); + let set_vectors = SetVectors { docid, embedder_id, _padding: [0; 3] }; + let payload_header = EntryHeader::SetVectors(set_vectors); let total_length = EntryHeader::total_set_vectors_size(embeddings.len(), dimensions); if total_length > max_grant { let mut value_file = tempfile::tempfile().map(BufWriter::new)?; @@ -650,9 +650,8 @@ impl<'b> ExtractorBbqueueSender<'b> { // to zero to allocate no extra space at all let dimensions = embedding.as_ref().map_or(0, |emb| emb.len()); - let hannoy_set_vector = - HannoySetVector { docid, embedder_id, extractor_id, _padding: [0; 2] }; - let payload_header = EntryHeader::HannoySetVector(hannoy_set_vector); + let set_vector = SetVector { docid, embedder_id, extractor_id, _padding: [0; 2] }; + let payload_header = EntryHeader::SetVector(set_vector); let total_length = EntryHeader::total_set_vector_size(dimensions); if total_length > max_grant { let mut value_file = tempfile::tempfile().map(BufWriter::new)?; diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 0547aa155..3655a59f6 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -67,7 +67,7 @@ where let mut bbbuffers = Vec::new(); let finished_extraction = AtomicBool::new(false); - let hannoy_memory = grenad_parameters.max_memory; + let vector_memory = grenad_parameters.max_memory; let (grenad_parameters, total_bbbuffer_capacity) = indexer_memory_settings(pool.current_num_threads(), grenad_parameters); @@ -132,7 +132,7 @@ where let vector_arroy = index.vector_store; let backend = index.get_vector_store(wtxn)?; - let hannoy_writers: Result> = embedders + let vector_stores: Result> = embedders .inner_as_ref() .iter() .map(|(embedder_name, runtime)| { @@ -155,10 +155,10 @@ where }) .collect(); - let mut hannoy_writers = hannoy_writers?; + let mut vector_stores = vector_stores?; let congestion = - write_to_db(writer_receiver, finished_extraction, index, wtxn, &hannoy_writers)?; + write_to_db(writer_receiver, finished_extraction, index, wtxn, &vector_stores)?; indexing_context.progress.update_progress(IndexingStep::WaitingForExtractors); @@ -172,8 +172,8 @@ where wtxn, indexing_context.progress, index_embeddings, - hannoy_memory, - &mut hannoy_writers, + vector_memory, + &mut vector_stores, None, &indexing_context.must_stop_processing, ) @@ -229,7 +229,7 @@ where let mut bbbuffers = Vec::new(); let finished_extraction = AtomicBool::new(false); - let hannoy_memory = grenad_parameters.max_memory; + let vector_memory = grenad_parameters.max_memory; let (grenad_parameters, total_bbbuffer_capacity) = indexer_memory_settings(pool.current_num_threads(), grenad_parameters); @@ -286,7 +286,7 @@ where let new_embedders = settings_delta.new_embedders(); let embedder_actions = settings_delta.embedder_actions(); let index_embedder_category_ids = settings_delta.new_embedder_category_id(); - let mut hannoy_writers = hannoy_writers_from_embedder_actions( + let mut vector_stores = vector_stores_from_embedder_actions( index, wtxn, embedder_actions, @@ -295,7 +295,7 @@ where )?; let congestion = - write_to_db(writer_receiver, finished_extraction, index, wtxn, &hannoy_writers)?; + write_to_db(writer_receiver, finished_extraction, index, wtxn, &vector_stores)?; indexing_context.progress.update_progress(IndexingStep::WaitingForExtractors); @@ -309,8 +309,8 @@ where wtxn, indexing_context.progress, index_embeddings, - hannoy_memory, - &mut hannoy_writers, + vector_memory, + &mut vector_stores, Some(embedder_actions), &indexing_context.must_stop_processing, ) @@ -340,7 +340,7 @@ where Ok(congestion) } -fn hannoy_writers_from_embedder_actions<'indexer>( +fn vector_stores_from_embedder_actions<'indexer>( index: &Index, rtxn: &RoTxn, embedder_actions: &'indexer BTreeMap, diff --git a/crates/milli/src/update/new/indexer/write.rs b/crates/milli/src/update/new/indexer/write.rs index 9cb014e25..55ccfdf35 100644 --- a/crates/milli/src/update/new/indexer/write.rs +++ b/crates/milli/src/update/new/indexer/write.rs @@ -23,7 +23,7 @@ pub fn write_to_db( finished_extraction: &AtomicBool, index: &Index, wtxn: &mut RwTxn<'_>, - hannoy_writers: &HashMap, + vector_stores: &HashMap, ) -> Result { // Used by by the HannoySetVector to copy the embedding into an // aligned memory area, required by arroy to accept a new vector. @@ -56,7 +56,7 @@ pub fn write_to_db( ReceiverAction::LargeVectors(large_vectors) => { let LargeVectors { docid, embedder_id, .. } = large_vectors; let (_, _, writer, dimensions) = - hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); + vector_stores.get(&embedder_id).expect("requested a missing embedder"); let mut embeddings = Embeddings::new(*dimensions); for embedding in large_vectors.read_embeddings(*dimensions) { embeddings.push(embedding.to_vec()).unwrap(); @@ -68,7 +68,7 @@ pub fn write_to_db( large_vector @ LargeVector { docid, embedder_id, extractor_id, .. }, ) => { let (_, _, writer, dimensions) = - hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); + vector_stores.get(&embedder_id).expect("requested a missing embedder"); let embedding = large_vector.read_embedding(*dimensions); writer.add_item_in_store(wtxn, docid, extractor_id, embedding)?; } @@ -80,12 +80,12 @@ pub fn write_to_db( &mut writer_receiver, index, wtxn, - hannoy_writers, + vector_stores, &mut aligned_embedding, )?; } - write_from_bbqueue(&mut writer_receiver, index, wtxn, hannoy_writers, &mut aligned_embedding)?; + write_from_bbqueue(&mut writer_receiver, index, wtxn, vector_stores, &mut aligned_embedding)?; Ok(ChannelCongestion { attempts: writer_receiver.sent_messages_attempts(), @@ -115,8 +115,8 @@ pub fn build_vectors( wtxn: &mut RwTxn<'_>, progress: &Progress, index_embeddings: Vec, - hannoy_memory: Option, - hannoy_writers: &mut HashMap, + vector_memory: Option, + vector_stores: &mut HashMap, embeder_actions: Option<&BTreeMap>, must_stop_processing: &MSP, ) -> Result<()> @@ -129,7 +129,7 @@ where let seed = rand::random(); let mut rng = rand::rngs::StdRng::seed_from_u64(seed); - for (_index, (embedder_name, _embedder, writer, dimensions)) in hannoy_writers { + for (_index, (embedder_name, _embedder, writer, dimensions)) in vector_stores { let dimensions = *dimensions; let is_being_quantized = embeder_actions .and_then(|actions| actions.get(*embedder_name).map(|action| action.is_being_quantized)) @@ -140,7 +140,7 @@ where &mut rng, dimensions, is_being_quantized, - hannoy_memory, + vector_memory, must_stop_processing, )?; } @@ -181,7 +181,7 @@ pub fn write_from_bbqueue( writer_receiver: &mut WriterBbqueueReceiver<'_>, index: &Index, wtxn: &mut RwTxn<'_>, - hannoy_writers: &HashMap, + vector_stores: &HashMap, aligned_embedding: &mut Vec, ) -> crate::Result<()> { while let Some(frame_with_header) = writer_receiver.recv_frame() { @@ -221,17 +221,17 @@ pub fn write_from_bbqueue( }, } } - EntryHeader::HannoyDeleteVector(HannoyDeleteVector { docid }) => { - for (_index, (_name, _embedder, writer, dimensions)) in hannoy_writers { + EntryHeader::DeleteVector(DeleteVector { docid }) => { + for (_index, (_name, _embedder, writer, dimensions)) in vector_stores { let dimensions = *dimensions; writer.del_items(wtxn, dimensions, docid)?; } } - EntryHeader::HannoySetVectors(asvs) => { - let HannoySetVectors { docid, embedder_id, .. } = asvs; + EntryHeader::SetVectors(asvs) => { + let SetVectors { docid, embedder_id, .. } = asvs; let frame = frame_with_header.frame(); let (_, _, writer, dimensions) = - hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); + vector_stores.get(&embedder_id).expect("requested a missing embedder"); let mut embeddings = Embeddings::new(*dimensions); let all_embeddings = asvs.read_all_embeddings_into_vec(frame, aligned_embedding); writer.del_items(wtxn, *dimensions, docid)?; @@ -245,12 +245,10 @@ pub fn write_from_bbqueue( writer.add_items(wtxn, docid, &embeddings)?; } } - EntryHeader::HannoySetVector( - asv @ HannoySetVector { docid, embedder_id, extractor_id, .. }, - ) => { + EntryHeader::SetVector(asv @ SetVector { docid, embedder_id, extractor_id, .. }) => { let frame = frame_with_header.frame(); let (_, _, writer, dimensions) = - hannoy_writers.get(&embedder_id).expect("requested a missing embedder"); + vector_stores.get(&embedder_id).expect("requested a missing embedder"); let embedding = asv.read_all_embeddings_into_vec(frame, aligned_embedding); if embedding.is_empty() { From f54773781a76bc4b1c83556f13d95a2b7302332b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 8 Sep 2025 15:00:02 +0200 Subject: [PATCH 57/62] Revert the fake 1.22 in index-scheduler as well --- crates/index-scheduler/src/upgrade/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index c4b281d54..4ca594936 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -43,7 +43,6 @@ pub fn upgrade_index_scheduler( (1, 17, _) => 0, (1, 18, _) => 0, (1, 19, _) => 0, - (1, 22, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) From cfb040e647fc73891dfbdba4e8e2d5504a36828b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 8 Sep 2025 16:41:48 +0200 Subject: [PATCH 58/62] remove extraneous space --- crates/meilisearch/src/routes/indexes/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 336740162..cc825f893 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -790,7 +790,7 @@ fn validate_settings( } if let Setting::Set(_) = &settings.vector_store { - features.check_vector_store_setting("setting `vectorStore` in the index settings")?; + features.check_vector_store_setting("setting `vectorStore` in the index settings")?; } Ok(settings.validate()?) From 6376571df05c3de3ef270dd7f3cab6b449a8719c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 8 Sep 2025 16:44:01 +0200 Subject: [PATCH 59/62] Add VectorStoreBackend to the list of components --- crates/meilisearch/src/routes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index fd6e777de..5a6780cbb 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -102,7 +102,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata, meilisearch_types::milli::vector::VectorStoreBackend)) )] pub struct MeilisearchApi; From a8cd81c7f480ab97d565d865e721b494c46b87cf Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 8 Sep 2025 16:53:57 +0200 Subject: [PATCH 60/62] get_vector_store returns an option, handles it in Index::settings --- crates/meilisearch-types/src/settings.rs | 5 ++++- crates/milli/src/index.rs | 9 ++++----- crates/milli/src/search/facet/filter_vector.rs | 2 +- crates/milli/src/search/new/vector_sort.rs | 2 +- crates/milli/src/search/similar.rs | 2 +- crates/milli/src/update/index_documents/mod.rs | 2 +- crates/milli/src/update/index_documents/transform.rs | 2 +- crates/milli/src/update/index_documents/typed_chunk.rs | 2 +- crates/milli/src/update/new/indexer/mod.rs | 6 +++--- crates/milli/src/update/new/vector_document.rs | 2 +- crates/milli/src/update/settings.rs | 2 +- 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index e1d314b8f..f73a66c4b 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -989,7 +989,10 @@ pub fn settings( facet_search: Setting::Set(facet_search), prefix_search: Setting::Set(prefix_search.unwrap_or_default()), chat: Setting::Set(chat), - vector_store: Setting::Set(vector_store), + vector_store: match vector_store { + Some(vector_store) => Setting::Set(vector_store), + None => Setting::Reset, + }, _kind: PhantomData, }; diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 54086d8a5..4f018e56f 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -469,12 +469,11 @@ impl Index { )?) } - pub fn get_vector_store(&self, rtxn: &RoTxn<'_>) -> Result { + pub fn get_vector_store(&self, rtxn: &RoTxn<'_>) -> Result> { Ok(self .main .remap_types::>() - .get(rtxn, main_key::VECTOR_STORE_BACKEND)? - .unwrap_or_default()) + .get(rtxn, main_key::VECTOR_STORE_BACKEND)?) } pub(crate) fn delete_vector_store(&self, wtxn: &mut RwTxn<'_>) -> Result { @@ -1799,7 +1798,7 @@ impl Index { ) -> Result> { let mut res = BTreeMap::new(); let embedders = self.embedding_configs(); - let backend = self.get_vector_store(rtxn)?; + let backend = self.get_vector_store(rtxn)?.unwrap_or_default(); for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); @@ -1828,7 +1827,7 @@ impl Index { pub fn vector_store_stats(&self, rtxn: &RoTxn<'_>) -> Result { let mut stats = VectorStoreStats::default(); let embedding_configs = self.embedding_configs(); - let backend = self.get_vector_store(rtxn)?; + let backend = self.get_vector_store(rtxn)?.unwrap_or_default(); for config in embedding_configs.embedding_configs(rtxn)? { let embedder_id = embedding_configs.embedder_id(rtxn, &config.name)?.unwrap(); diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 2e34eb068..278179586 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -82,7 +82,7 @@ fn evaluate_inner( embedding_configs: &[IndexEmbeddingConfig], filter: &VectorFilter<'_>, ) -> crate::Result { - let backend = index.get_vector_store(rtxn)?; + let backend = index.get_vector_store(rtxn)?.unwrap_or_default(); let embedder_name = embedder.value(); let available_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index ecd61e322..5da4c7145 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -54,7 +54,7 @@ impl VectorSort { vector_candidates: &RoaringBitmap, ) -> Result<()> { let target = &self.target; - let backend = ctx.index.get_vector_store(ctx.txn)?; + let backend = ctx.index.get_vector_store(ctx.txn)?.unwrap_or_default(); let before = Instant::now(); let reader = diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index fb2e8103d..b4933e1d0 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -72,7 +72,7 @@ impl<'a> Similar<'a> { crate::UserError::InvalidSimilarEmbedder(self.embedder_name.to_owned()) })?; - let backend = self.index.get_vector_store(self.rtxn)?; + let backend = self.index.get_vector_store(self.rtxn)?.unwrap_or_default(); let reader = VectorStore::new(backend, self.index.vector_store, embedder_index, self.quantized); diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index ea468f08b..205f8ef5d 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -485,7 +485,7 @@ where // If an embedder wasn't used in the typedchunk but must be binary quantized // we should insert it in `dimension` - let backend = self.index.get_vector_store(self.wtxn)?; + let backend = self.index.get_vector_store(self.wtxn)?.unwrap_or_default(); for (name, action) in settings_diff.embedding_config_updates.iter() { if action.is_being_quantized && !dimension.contains_key(name.as_str()) { let index = self.index.embedding_configs().embedder_id(self.wtxn, name)?.ok_or( diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index 8df8722c3..aa100f2d5 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -834,7 +834,7 @@ impl<'a, 'i> Transform<'a, 'i> { None }; - let backend = self.index.get_vector_store(wtxn)?; + let backend = self.index.get_vector_store(wtxn)?.unwrap_or_default(); let readers: BTreeMap<&str, (VectorStore, &RoaringBitmap)> = settings_diff .embedding_config_updates .iter() diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index 228dc4797..eb2a0799b 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -619,7 +619,7 @@ pub(crate) fn write_typed_chunk_into_index( let _entered = span.enter(); let embedders = index.embedding_configs(); - let backend = index.get_vector_store(wtxn)?; + let backend = index.get_vector_store(wtxn)?.unwrap_or_default(); let mut remove_vectors_builder = MergerBuilder::new(KeepFirst); let mut manual_vectors_builder = MergerBuilder::new(KeepFirst); diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index 3655a59f6..f613ad0b6 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -131,7 +131,7 @@ where let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map); let vector_arroy = index.vector_store; - let backend = index.get_vector_store(wtxn)?; + let backend = index.get_vector_store(wtxn)?.unwrap_or_default(); let vector_stores: Result> = embedders .inner_as_ref() .iter() @@ -348,7 +348,7 @@ fn vector_stores_from_embedder_actions<'indexer>( index_embedder_category_ids: &'indexer std::collections::HashMap, ) -> Result> { let vector_arroy = index.vector_store; - let backend = index.get_vector_store(rtxn)?; + let backend = index.get_vector_store(rtxn)?.unwrap_or_default(); embedders .inner_as_ref() @@ -390,7 +390,7 @@ fn delete_old_embedders_and_fragments( where SD: SettingsDelta, { - let backend = index.get_vector_store(wtxn)?; + let backend = index.get_vector_store(wtxn)?.unwrap_or_default(); for action in settings_delta.embedder_actions().values() { let Some(WriteBackToDocuments { embedder_id, .. }) = action.write_back() else { continue; diff --git a/crates/milli/src/update/new/vector_document.rs b/crates/milli/src/update/new/vector_document.rs index bff407b9f..f74d79d47 100644 --- a/crates/milli/src/update/new/vector_document.rs +++ b/crates/milli/src/update/new/vector_document.rs @@ -120,7 +120,7 @@ impl<'t> VectorDocumentFromDb<'t> { config: &IndexEmbeddingConfig, status: &EmbeddingStatus, ) -> Result> { - let backend = self.index.get_vector_store(self.rtxn)?; + let backend = self.index.get_vector_store(self.rtxn)?.unwrap_or_default(); let reader = VectorStore::new( backend, self.index.vector_store, diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index bd6e6c301..b8a8fed1c 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -1506,7 +1506,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { where MSP: Fn() -> bool + Sync, { - let old_backend = self.index.get_vector_store(self.wtxn)?; + let old_backend = self.index.get_vector_store(self.wtxn)?.unwrap_or_default(); let new_backend = match self.vector_store { Setting::Set(new_backend) => { From 16461a91457310411645eb9216849d49c61a1a5f Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 9 Sep 2025 14:52:43 +0200 Subject: [PATCH 61/62] add unit test --- .../tests/settings/get_settings.rs | 2 +- crates/meilisearch/tests/vector/mod.rs | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index d9bb78d6c..ae0c37048 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -197,7 +197,7 @@ test_setting_routes!( { setting: vector_store, update_verb: patch, - default_value: "stable" + default_value: null }, ); diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index d89fbf0a9..0ed4f4d3c 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -6,11 +6,13 @@ mod openai; mod rest; mod settings; +use std::collections::HashMap; use std::str::FromStr; use meili_snap::{json_string, snapshot}; use meilisearch::option::MaxThreads; pub use rest::create_mock; +use serde_json::Value; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; @@ -781,3 +783,90 @@ async fn add_remove_one_vector_4588() { } "#); } + +#[actix_rt::test] +async fn change_backend() { + let server = Server::new().await; + let index = server.unique_index(); + server.set_features(json!({"vectorStoreSetting": true})).await; + let (response, code) = index + .update_settings(json!({ + "vectorStore": "stable" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (response, code) = index + .update_settings(json!({ + "embedders": { + "manual": { + "source": "userProvided", + "dimensions": 3, + "binaryQuantized": false, + } + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let documents = json!([ + {"id": 0, "name": "kefir", "_vectors": { "manual": [-1.2, -2.3, 3.2] }}, + {"id": 1, "name": "echo", "_vectors": { "manual": [2.5, 1.5, -130] }}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + server.wait_task(value.uid()).await.succeeded(); + + // Make sure the documents are binary quantized + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + let stable_embeddings: HashMap = documents["results"] + .as_array() + .unwrap() + .iter() + .map(|o| (o["id"].clone(), o["_vectors"]["manual"]["embeddings"].clone())) + .collect(); + + let (response, code) = index + .update_settings(json!({ + "vectorStore": "experimental" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + + let experimental_embeddings: HashMap = documents["results"] + .as_array() + .unwrap() + .iter() + .map(|o| (o["id"].clone().clone(), o["_vectors"]["manual"]["embeddings"].clone())) + .collect(); + + let (response, code) = index + .update_settings(json!({ + "vectorStore": "stable" + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let (documents, _code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + let back_to_stable_embeddings: HashMap = documents["results"] + .as_array() + .unwrap() + .iter() + .map(|o| (o["id"].clone(), o["_vectors"]["manual"]["embeddings"].clone())) + .collect(); + + assert_eq!(stable_embeddings, experimental_embeddings); + assert_eq!(experimental_embeddings, back_to_stable_embeddings); +} From 4cc8fb2c5c775210560d8adfff010d420b97cd53 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 9 Sep 2025 17:42:33 +0200 Subject: [PATCH 62/62] Add comment about upgrade procedure Co-authored-by: Tamo --- crates/milli/src/vector/store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/milli/src/vector/store.rs b/crates/milli/src/vector/store.rs index 11385a0e4..d4d3f26cb 100644 --- a/crates/milli/src/vector/store.rs +++ b/crates/milli/src/vector/store.rs @@ -1056,6 +1056,7 @@ impl VectorStore { let dimensions = arroy_reader.dimensions(); let hannoy_writer: hannoy::Writer = hannoy::Writer::new(self.database.remap_types(), index, dimensions); + // Since the bq mode of arroy and hannoy are not compatible, we have to clear and re-insert everything hannoy_writer.clear(hannoy_wtxn)?; for entry in arroy_reader.iter(arroy_rtxn)? { let (item, mut vector) = entry?; @@ -1093,6 +1094,7 @@ impl VectorStore { let dimensions = hannoy_reader.dimensions(); let arroy_writer: arroy::Writer = arroy::Writer::new(self.database.remap_types(), index, dimensions); + // Since the bq mode of arroy and hannoy are not compatible, we have to clear and re-insert everything arroy_writer.clear(arroy_wtxn)?; for entry in hannoy_reader.iter(hannoy_rtxn)? { let (item, mut vector) = entry?;