From a9bb64c55a285b9cf90d29e85f9d622cbf893887 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 15:28:10 +0200 Subject: [PATCH 01/76] Unrelated minor fixes --- crates/meilisearch/src/routes/indexes/documents.rs | 2 -- crates/milli/src/filterable_attributes_rules.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index a93d736f7..bc5539081 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1461,8 +1461,6 @@ fn some_documents<'a, 't: 'a>( document.remove("_vectors"); } RetrieveVectors::Retrieve => { - // Clippy is simply wrong - #[allow(clippy::manual_unwrap_or_default)] let mut vectors = match document.remove("_vectors") { Some(Value::Object(map)) => map, _ => Default::default(), diff --git a/crates/milli/src/filterable_attributes_rules.rs b/crates/milli/src/filterable_attributes_rules.rs index ae1a9755a..5ba8a99d8 100644 --- a/crates/milli/src/filterable_attributes_rules.rs +++ b/crates/milli/src/filterable_attributes_rules.rs @@ -111,7 +111,7 @@ impl FilterableAttributesFeatures { self.filter.is_filterable_null() } - /// Check if `IS EXISTS` is allowed + /// Check if `EXISTS` is allowed pub fn is_filterable_exists(&self) -> bool { self.filter.is_filterable_exists() } From 20525376815e6c0d6962744701f954bf9f03f7cb Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 15:28:35 +0200 Subject: [PATCH 02/76] Implement core filter logic --- crates/milli/src/index.rs | 2 +- crates/milli/src/search/facet/filter.rs | 15 ++- .../milli/src/search/facet/filter_vector.rs | 123 ++++++++++++++++++ crates/milli/src/search/facet/mod.rs | 1 + .../src/update/index_documents/transform.rs | 2 +- 5 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 crates/milli/src/search/facet/filter_vector.rs diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index b2ec992ba..2751498bf 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1776,7 +1776,7 @@ impl Index { embedder_info.embedder_id, config.config.quantized(), ); - let embeddings = reader.item_vectors(rtxn, docid)?; + let embeddings = reader.item_vectors(rtxn, docid)?; // MARKER res.insert( config.name.to_owned(), (embeddings, embedder_info.embedding_status.must_regenerate(docid)), diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c3eba8031..f80d1681f 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -10,7 +10,7 @@ use memchr::memmem::Finder; use roaring::{MultiOps, RoaringBitmap}; use serde_json::Value; -use super::facet_range_search; +use super::{facet_range_search, filter_vector::VectorFilter}; use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; @@ -234,8 +234,11 @@ impl<'a> Filter<'a> { pub fn evaluate(&self, rtxn: &heed::RoTxn<'_>, index: &Index) -> Result { // to avoid doing this for each recursive call we're going to do it ONCE ahead of time let fields_ids_map = index.fields_ids_map(rtxn)?; - let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; + let filterable_attributes_rules = dbg!(index.filterable_attributes_rules(rtxn)?); + for fid in self.condition.fids(MAX_FILTER_DEPTH) { + println!("{fid:?}"); + let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) @@ -542,7 +545,13 @@ impl<'a> Filter<'a> { .union() } FilterCondition::Condition { fid, op } => { - let Some(field_id) = field_ids_map.id(fid.value()) else { + let value = fid.value(); + if VectorFilter::matches(value, op) { + let vector_filter = VectorFilter::parse(value)?; + return vector_filter.evaluate(rtxn, index, universe); + } + + let Some(field_id) = field_ids_map.id(value) else { return Ok(RoaringBitmap::new()); }; let Some((rule_index, features)) = diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs new file mode 100644 index 000000000..701ab561c --- /dev/null +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -0,0 +1,123 @@ +use filter_parser::Condition; +use roaring::RoaringBitmap; + +use crate::error::{Error, UserError}; +use crate::vector::{ArroyStats, ArroyWrapper}; +use crate::{Index, Result}; + +pub(super) struct VectorFilter<'a> { + embedder_name: &'a str, + fragment_name: Option<&'a str>, + user_provided: bool, + // TODO: not_user_provided: bool, +} + +impl<'a> VectorFilter<'a> { + pub(super) fn matches(value: &str, op: &Condition) -> bool { + matches!(op, Condition::Exists) && value.starts_with("_vectors.") + } + + /// Parses a vector filter string. + /// + /// Valid formats: + /// - `_vectors.{embedder_name}` + /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.userProvided` + pub(super) fn parse(s: &'a str) -> Result { + let mut split = s.split('.').peekable(); + + if split.next() != Some("_vectors") { + return Err(Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter must start with '_vectors'", + )))); + } + + let embedder_name = split.next().ok_or_else(|| { + Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter must contain an embedder name", + ))) + })?; + + let mut fragment_name = None; + if split.peek() == Some(&"fragments") { + split.next(); + + fragment_name = Some(split.next().ok_or_else(|| { + Error::UserError(UserError::InvalidFilter( + String::from("Vector filter is inconsistent: either specify a fragment name or remove the 'fragments' part"), + )) + })?); + } + + let mut user_provided = false; + if split.peek() == Some(&"userProvided") || split.peek() == Some(&"user_provided") { + split.next(); + user_provided = true; + } + + if let Some(next) = split.next() { + return Err(Error::UserError(UserError::InvalidFilter(format!( + "Unexpected part in vector filter: '{next}'" + )))); + } + + Ok(Self { embedder_name, fragment_name, user_provided }) + } + + pub(super) fn evaluate( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + universe: Option<&RoaringBitmap>, + ) -> Result { + let index_embedding_configs = index.embedding_configs(); + let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; + + let Some(embedder_config) = + embedding_configs.iter().find(|config| config.name == self.embedder_name) + else { + return Ok(RoaringBitmap::new()); + }; + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, self.embedder_name)? + else { + return Ok(RoaringBitmap::new()); + }; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedder_config.config.quantized(), + ); + + let mut docids = if let Some(fragment_name) = self.fragment_name { + let Some(fragment_config) = embedder_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + else { + return Ok(RoaringBitmap::new()); + }; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } else { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + }; + + // FIXME: performance + if self.user_provided { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + docids &= user_provided_docsids; + } + + if let Some(universe) = universe { + docids &= universe; + } + + Ok(docids) + } +} diff --git a/crates/milli/src/search/facet/mod.rs b/crates/milli/src/search/facet/mod.rs index a5e65c95d..fac85df59 100644 --- a/crates/milli/src/search/facet/mod.rs +++ b/crates/milli/src/search/facet/mod.rs @@ -17,6 +17,7 @@ mod facet_range_search; mod facet_sort_ascending; mod facet_sort_descending; mod filter; +mod filter_vector; mod search; fn facet_extreme_value<'t>( diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index e07483aff..d69768d4b 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -966,7 +966,7 @@ impl<'a, 'i> Transform<'a, 'i> { // some user provided, remove only the ids that are not user provided let to_delete = arroy.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() - })?; + })?; // MARKER for to_delete in to_delete { arroy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; From 9c60e9689f806442abd9e9ebe40f98ba17c65db9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 18:34:24 +0200 Subject: [PATCH 03/76] Support not specifying an embedder in the vector filter --- .../milli/src/search/facet/filter_vector.rs | 95 +++++++++++-------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 701ab561c..ee1c19923 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -6,7 +6,7 @@ use crate::vector::{ArroyStats, ArroyWrapper}; use crate::{Index, Result}; pub(super) struct VectorFilter<'a> { - embedder_name: &'a str, + embedder_name: Option<&'a str>, fragment_name: Option<&'a str>, user_provided: bool, // TODO: not_user_provided: bool, @@ -14,12 +14,14 @@ pub(super) struct VectorFilter<'a> { impl<'a> VectorFilter<'a> { pub(super) fn matches(value: &str, op: &Condition) -> bool { - matches!(op, Condition::Exists) && value.starts_with("_vectors.") + matches!(op, Condition::Exists) && (value.starts_with("_vectors.") || value == "_vectors") } /// Parses a vector filter string. /// /// Valid formats: + /// - `_vectors` + /// - `_vectors.userProvided` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` @@ -33,11 +35,7 @@ impl<'a> VectorFilter<'a> { )))); } - let embedder_name = split.next().ok_or_else(|| { - Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter must contain an embedder name", - ))) - })?; + let embedder_name = split.next(); let mut fragment_name = None; if split.peek() == Some(&"fragments") { @@ -74,44 +72,63 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let Some(embedder_config) = - embedding_configs.iter().find(|config| config.name == self.embedder_name) - else { - return Ok(RoaringBitmap::new()); - }; - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, self.embedder_name)? - else { - return Ok(RoaringBitmap::new()); - }; - - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedder_config.config.quantized(), - ); - - let mut docids = if let Some(fragment_name) = self.fragment_name { - let Some(fragment_config) = embedder_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) + let mut embedders = Vec::new(); + if let Some(embedder_name) = self.embedder_name { + let Some(embedder_config) = + embedding_configs.iter().find(|config| config.name == embedder_name) else { return Ok(RoaringBitmap::new()); }; - - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, embedder_name)? + else { + return Ok(RoaringBitmap::new()); + }; + + embedders.push((embedder_config, embedder_info)); } else { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents + for embedder_config in embedding_configs.iter() { + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, &embedder_config.name)? + else { + continue; + }; + embedders.push((embedder_config, embedder_info)); + } }; + + let mut docids = RoaringBitmap::new(); + for (embedder_config, embedder_info) in embedders { + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedder_config.config.quantized(), + ); - // FIXME: performance - if self.user_provided { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - docids &= user_provided_docsids; + let mut new_docids = if let Some(fragment_name) = self.fragment_name { + let Some(fragment_config) = embedder_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + else { + return Ok(RoaringBitmap::new()); + }; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } else { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + }; + + // FIXME: performance + if self.user_provided { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + new_docids &= user_provided_docsids; + } + + docids |= new_docids; } if let Some(universe) = universe { From 5cced0af02018067d78f85b1348ede9b2672eb86 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 18:41:03 +0200 Subject: [PATCH 04/76] Prevent having both a fragment name and userProvided --- crates/milli/src/search/facet/filter_vector.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index ee1c19923..9a0a50124 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -25,7 +25,6 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.userProvided` pub(super) fn parse(s: &'a str) -> Result { let mut split = s.split('.').peekable(); @@ -54,6 +53,12 @@ impl<'a> VectorFilter<'a> { user_provided = true; } + if fragment_name.is_some() && user_provided { + return Err(Error::UserError(UserError::InvalidFilter( + String::from("Vector filter cannot specify both a fragment name and userProvided"), + ))); + } + if let Some(next) = split.next() { return Err(Error::UserError(UserError::InvalidFilter(format!( "Unexpected part in vector filter: '{next}'" From 40e7284d70865600e7832a10d357bad9ae68cec5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 10:01:35 +0200 Subject: [PATCH 05/76] Add tests --- crates/meilisearch/tests/search/filters.rs | 162 +++++++++++++++++++ crates/meilisearch/tests/vector/fragments.rs | 2 +- crates/meilisearch/tests/vector/mod.rs | 1 + 3 files changed, 164 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index ffa025f5c..361762b6c 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -731,3 +731,165 @@ async fn test_filterable_attributes_priority() { ) .await; } + +#[actix_rt::test] +async fn test_vector_filter() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.other EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); + + // This one is counterintuitive, but it is the same as the previous one. + // It's because userProvided is interpreted as an embedder name + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 1 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 1 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.fragments.basic EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); +} diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 2626284a0..3ce452c1f 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -10,7 +10,7 @@ use crate::common::{Owned, Shared}; use crate::json; use crate::vector::{GetAllDocumentsOptions, Server}; -async fn shared_index_for_fragments() -> Index<'static, Shared> { +pub async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); let (server, uid) = INDEX .get_or_init(|| async { diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 7f54489b6..9ba37cae3 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,6 +14,7 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; +pub use fragments::shared_index_for_fragments; async fn get_server_vector() -> Server { Server::new().await From 2d45124d9b379fd73dba54e7d56d4f067eeb477c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 10:01:50 +0200 Subject: [PATCH 06/76] Fix parsing --- crates/milli/src/search/facet/filter.rs | 9 +++++++-- crates/milli/src/search/facet/filter_vector.rs | 6 ++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index f80d1681f..c9728966a 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -241,7 +241,7 @@ impl<'a> Filter<'a> { let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) - .is_some_and(|(_, features)| features.is_filterable()) + .is_some_and(|(_, features)| features.is_filterable()) || VectorFilter::matches(attribute) { continue; } @@ -546,7 +546,12 @@ impl<'a> Filter<'a> { } FilterCondition::Condition { fid, op } => { let value = fid.value(); - if VectorFilter::matches(value, op) { + if VectorFilter::matches(value) { + if !matches!(op, Condition::Exists) { + return Err(Error::UserError(UserError::InvalidFilter( + String::from("Vector filter can only be used with the `exists` operator"), + ))); + } let vector_filter = VectorFilter::parse(value)?; return vector_filter.evaluate(rtxn, index, universe); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 9a0a50124..473741f14 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,4 +1,3 @@ -use filter_parser::Condition; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; @@ -13,15 +12,14 @@ pub(super) struct VectorFilter<'a> { } impl<'a> VectorFilter<'a> { - pub(super) fn matches(value: &str, op: &Condition) -> bool { - matches!(op, Condition::Exists) && (value.starts_with("_vectors.") || value == "_vectors") + pub(super) fn matches(value: &str) -> bool { + value.starts_with("_vectors.") || value == "_vectors" } /// Parses a vector filter string. /// /// Valid formats: /// - `_vectors` - /// - `_vectors.userProvided` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` From 0301d8f239f9d860c663bec9b48b4b08477e7885 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:39:10 +0200 Subject: [PATCH 07/76] Improve error handling --- crates/filter-parser/src/lib.rs | 113 +++++++++- crates/meilisearch/tests/search/filters.rs | 109 ++++++---- crates/milli/src/search/facet/filter.rs | 11 +- .../milli/src/search/facet/filter_vector.rs | 196 ++++++++++++++---- 4 files changed, 342 insertions(+), 87 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 938702103..b64477170 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -60,7 +60,7 @@ use nom::combinator::{cut, eof, map, opt}; use nom::multi::{many0, separated_list1}; use nom::number::complete::recognize_float; use nom::sequence::{delimited, preceded, terminated, tuple}; -use nom::Finish; +use nom::{Finish, Slice}; use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; @@ -121,6 +121,16 @@ impl<'a> Token<'a> { Err(Error::new_from_kind(self.span, ErrorKind::NonFiniteFloat)) } } + + /// Split the token by a delimiter and return an iterator of tokens. + /// Each token in the iterator will have its own span that corresponds to a slice of the original token's span. + pub fn split(&self, delimiter: &'a str) -> impl Iterator> + '_ { + let original_addr = self.value().as_ptr() as usize; + self.value().split(delimiter).map(move |part| { + let offset = part.as_ptr() as usize - original_addr; + Token::new(self.span.slice(offset..offset + part.len()), Some(part.to_string())) + }) + } } impl<'a> From> for Token<'a> { @@ -604,6 +614,8 @@ impl std::fmt::Display for Token<'_> { #[cfg(test)] pub mod tests { + use std::fmt::format; + use FilterCondition as Fc; use super::*; @@ -1043,4 +1055,103 @@ pub mod tests { let token: Token = s.into(); assert_eq!(token.value(), s); } + + #[test] + fn split() { + let s = "test string that should not be parsed\n newline"; + let token: Token = s.into(); + let parts: Vec<_> = token.split(" ").collect(); + insta::assert_snapshot!(format!("{parts:#?}"), @r#" + [ + Token { + span: LocatedSpan { + offset: 0, + line: 1, + fragment: "test", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "test", + ), + }, + Token { + span: LocatedSpan { + offset: 5, + line: 1, + fragment: "string", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "string", + ), + }, + Token { + span: LocatedSpan { + offset: 12, + line: 1, + fragment: "that", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "that", + ), + }, + Token { + span: LocatedSpan { + offset: 17, + line: 1, + fragment: "should", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "should", + ), + }, + Token { + span: LocatedSpan { + offset: 24, + line: 1, + fragment: "not", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "not", + ), + }, + Token { + span: LocatedSpan { + offset: 28, + line: 1, + fragment: "be", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "be", + ), + }, + Token { + span: LocatedSpan { + offset: 31, + line: 1, + fragment: "parsed\n", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "parsed\n", + ), + }, + Token { + span: LocatedSpan { + offset: 39, + line: 2, + fragment: "newline", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "newline", + ), + }, + ] + "#); + } } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 361762b6c..384605ca6 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -736,10 +736,12 @@ async fn test_filterable_attributes_priority() { async fn test_vector_filter() { let index = crate::vector::shared_index_for_fragments().await; - let (value, _code) = index.search_post(json!({ - "filter": "_vectors EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -764,42 +766,44 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.other EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.other EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { - "hits": [], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 0 + "message": "Index `[uuid]`: The embedder `other` does not exist. Available embedders are: `rest`.\n10:15 _vectors.other EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); - + // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.userProvided EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { - "hits": [], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 0 + "message": "Index `[uuid]`: The embedder `userProvided` does not exist. Available embedders are: `rest`.\n10:22 _vectors.userProvided EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -824,10 +828,12 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.userProvided EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -843,10 +849,12 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.fragments.withBreed EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -864,11 +872,13 @@ async fn test_vector_filter() { "estimatedTotalHits": 2 } "#); - - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.fragments.basic EXISTS", - "attributesToRetrieve": ["id"] - })).await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.basic EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -892,4 +902,19 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.other EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: The fragment `other` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`.\n25:30 _vectors.rest.fragments.other EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c9728966a..c0419997c 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -241,7 +241,8 @@ impl<'a> Filter<'a> { let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) - .is_some_and(|(_, features)| features.is_filterable()) || VectorFilter::matches(attribute) + .is_some_and(|(_, features)| features.is_filterable()) + || VectorFilter::matches(attribute) { continue; } @@ -548,11 +549,11 @@ impl<'a> Filter<'a> { let value = fid.value(); if VectorFilter::matches(value) { if !matches!(op, Condition::Exists) { - return Err(Error::UserError(UserError::InvalidFilter( - String::from("Vector filter can only be used with the `exists` operator"), - ))); + return Err(Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter can only be used with the `exists` operator", + )))); } - let vector_filter = VectorFilter::parse(value)?; + let vector_filter = VectorFilter::parse(fid)?; return vector_filter.evaluate(rtxn, index, universe); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 473741f14..0a7ac313a 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,16 +1,118 @@ +use filter_parser::Token; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; use crate::vector::{ArroyStats, ArroyWrapper}; -use crate::{Index, Result}; +use crate::Index; pub(super) struct VectorFilter<'a> { - embedder_name: Option<&'a str>, - fragment_name: Option<&'a str>, + embedder_token: Option>, + fragment_token: Option>, user_provided: bool, // TODO: not_user_provided: bool, } +#[derive(Debug)] +pub enum VectorFilterError<'a> { + EmptyFilter, + InvalidPrefix(Token<'a>), + MissingFragmentName(Token<'a>), + UserProvidedWithFragment(Token<'a>), + LeftoverToken(Token<'a>), + EmbedderDoesNotExist { + embedder: &'a Token<'a>, + available: Vec, + }, + FragmentDoesNotExist { + embedder: &'a Token<'a>, + fragment: &'a Token<'a>, + available: Vec, + }, +} + +use VectorFilterError::*; + +impl std::error::Error for VectorFilterError<'_> {} + +impl std::fmt::Display for VectorFilterError<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EmptyFilter => { + write!(f, "Vector filter cannot be empty.") + } + InvalidPrefix(prefix) => { + write!( + f, + "Vector filter must start with `_vectors` but found `{}`.", + prefix.value() + ) + } + MissingFragmentName(_token) => { + write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") + } + UserProvidedWithFragment(_token) => { + write!(f, "Vector filter cannot specify both a fragment name and userProvided.") + } + LeftoverToken(token) => { + write!(f, "Vector filter has leftover token: `{}`.", token.value()) + } + EmbedderDoesNotExist { embedder, available } => { + write!(f, "The embedder `{}` does not exist.", embedder.value())?; + if available.is_empty() { + write!(f, " This index does not have configured embedders.") + } else { + write!(f, " Available embedders are: ")?; + let mut available = available.clone(); + available.sort_unstable(); + for (idx, embedder) in available.iter().enumerate() { + write!(f, "`{embedder}`")?; + if idx != available.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, ".") + } + } + FragmentDoesNotExist { embedder, fragment, available } => { + write!( + f, + "The fragment `{}` does not exist on embedder `{}`.", + fragment.value(), + embedder.value(), + )?; + if available.is_empty() { + write!(f, " This embedder does not have configured fragments.") + } else { + write!(f, " Available fragments on this embedder are: ")?; + let mut available = available.clone(); + available.sort_unstable(); + for (idx, fragment) in available.iter().enumerate() { + write!(f, "`{fragment}`")?; + if idx != available.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, ".") + } + } + } + } +} + +impl<'a> From> for Error { + fn from(err: VectorFilterError<'a>) -> Self { + match &err { + EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), + InvalidPrefix(token) + | MissingFragmentName(token) + | UserProvidedWithFragment(token) + | LeftoverToken(token) => token.clone().as_external_error(err).into(), + EmbedderDoesNotExist { embedder: token, .. } + | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), + } + } +} + impl<'a> VectorFilter<'a> { pub(super) fn matches(value: &str) -> bool { value.starts_with("_vectors.") || value == "_vectors" @@ -23,71 +125,74 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - pub(super) fn parse(s: &'a str) -> Result { - let mut split = s.split('.').peekable(); + pub(super) fn parse(s: &'a Token<'a>) -> Result> { + let mut split = s.split(".").peekable(); - if split.next() != Some("_vectors") { - return Err(Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter must start with '_vectors'", - )))); + match split.next() { + Some(token) if token.value() == "_vectors" => (), + Some(token) => return Err(InvalidPrefix(token)), + None => return Err(EmptyFilter), } let embedder_name = split.next(); let mut fragment_name = None; - if split.peek() == Some(&"fragments") { - split.next(); + if split.peek().map(|t| t.value()) == Some("fragments") { + let token = split.next().expect("it was peeked before"); - fragment_name = Some(split.next().ok_or_else(|| { - Error::UserError(UserError::InvalidFilter( - String::from("Vector filter is inconsistent: either specify a fragment name or remove the 'fragments' part"), - )) - })?); + fragment_name = Some(split.next().ok_or(MissingFragmentName(token))?); } - let mut user_provided = false; - if split.peek() == Some(&"userProvided") || split.peek() == Some(&"user_provided") { - split.next(); - user_provided = true; + let mut user_provided_token = None; + if split.peek().map(|t| t.value()) == Some("userProvided") + || split.peek().map(|t| t.value()) == Some("user_provided") + { + user_provided_token = split.next(); } - if fragment_name.is_some() && user_provided { - return Err(Error::UserError(UserError::InvalidFilter( - String::from("Vector filter cannot specify both a fragment name and userProvided"), - ))); + if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { + return Err(UserProvidedWithFragment(user_provided_token.clone()))?; } if let Some(next) = split.next() { - return Err(Error::UserError(UserError::InvalidFilter(format!( - "Unexpected part in vector filter: '{next}'" - )))); + return Err(LeftoverToken(next))?; } - Ok(Self { embedder_name, fragment_name, user_provided }) + Ok(Self { + embedder_token: embedder_name, + fragment_token: fragment_name, + user_provided: user_provided_token.is_some(), + }) } pub(super) fn evaluate( - &self, + self, rtxn: &heed::RoTxn<'_>, index: &Index, universe: Option<&RoaringBitmap>, - ) -> Result { + ) -> crate::Result { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; let mut embedders = Vec::new(); - if let Some(embedder_name) = self.embedder_name { + if let Some(embedder_token) = &self.embedder_token { + let embedder_name = embedder_token.value(); let Some(embedder_config) = embedding_configs.iter().find(|config| config.name == embedder_name) else { - return Ok(RoaringBitmap::new()); + return Err(EmbedderDoesNotExist { + embedder: embedder_token, + available: embedding_configs.iter().map(|c| c.name.clone()).collect(), + })?; }; - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, embedder_name)? + let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? else { - return Ok(RoaringBitmap::new()); + return Err(EmbedderDoesNotExist { + embedder: embedder_token, + available: embedding_configs.iter().map(|c| c.name.clone()).collect(), + })?; }; - + embedders.push((embedder_config, embedder_info)); } else { for embedder_config in embedding_configs.iter() { @@ -99,7 +204,7 @@ impl<'a> VectorFilter<'a> { embedders.push((embedder_config, embedder_info)); } }; - + let mut docids = RoaringBitmap::new(); for (embedder_config, embedder_info) in embedders { let arroy_wrapper = ArroyWrapper::new( @@ -108,14 +213,27 @@ impl<'a> VectorFilter<'a> { embedder_config.config.quantized(), ); - let mut new_docids = if let Some(fragment_name) = self.fragment_name { + let mut new_docids = if let Some(fragment_token) = &self.fragment_token { + let fragment_name = fragment_token.value(); let Some(fragment_config) = embedder_config .fragments .as_slice() .iter() .find(|fragment| fragment.name == fragment_name) else { - return Ok(RoaringBitmap::new()); + return Err(FragmentDoesNotExist { + embedder: self + .embedder_token + .as_ref() + .expect("there can't be a fragment without an embedder"), + fragment: fragment_token, + available: embedder_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; }; arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? From d43cd40807f87624bb4a7c87c44830b01f887458 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:48:23 +0200 Subject: [PATCH 08/76] Split tests --- crates/meilisearch/tests/search/filters.rs | 98 ++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 384605ca6..1c49fa5e0 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -733,29 +733,29 @@ async fn test_filterable_attributes_priority() { } #[actix_rt::test] -async fn test_vector_filter() { +async fn vector_filter_all_embedders() { let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -765,11 +765,16 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_non_existant_embedder() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.other EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -780,13 +785,18 @@ async fn test_vector_filter() { "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); +} + +#[actix_rt::test] +async fn vector_filter_all_embedders_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name let (value, _code) = index .search_post(json!({ "filter": "_vectors.userProvided EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -797,27 +807,32 @@ async fn test_vector_filter() { "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); +} + +#[actix_rt::test] +async fn vector_filter_specific_embedder() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -827,18 +842,23 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.userProvided EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 1 + "name": "echo" } ], "query": "", @@ -848,21 +868,26 @@ async fn test_vector_filter() { "estimatedTotalHits": 1 } "#); +} + +#[actix_rt::test] +async fn vector_filter_specific_fragment() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.withBreed EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -876,23 +901,23 @@ async fn test_vector_filter() { let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.basic EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -902,11 +927,16 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_non_existant_fragment() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.other EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -918,3 +948,23 @@ async fn test_vector_filter() { } "#); } + +#[actix_rt::test] +async fn vector_filter_specific_fragment_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.other.userProvided EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter cannot specify both a fragment name and userProvided.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); +} From b4cafec8b3941b2add4fe4ce99047b1b28da18bf Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:56:19 +0200 Subject: [PATCH 09/76] Add tests for operators along vector filter --- crates/meilisearch/tests/search/filters.rs | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 1c49fa5e0..2e71b5435 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -968,3 +968,67 @@ async fn vector_filter_specific_fragment_user_provided() { } "#); } + +#[actix_rt::test] +async fn vector_filter_negation() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.userProvided NOT EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); +} + +#[actix_rt::test] +async fn vector_filter_or_combination() { +let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS OR _vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "echo" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); +} From 29b74424ad6313ea673869a658e10883fca7e4b1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 12:03:32 +0200 Subject: [PATCH 10/76] Clean code --- crates/meilisearch/tests/search/filters.rs | 2 +- crates/milli/src/index.rs | 2 +- crates/milli/src/search/facet/filter.rs | 4 +--- crates/milli/src/search/facet/filter_vector.rs | 1 - crates/milli/src/update/index_documents/transform.rs | 2 +- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 2e71b5435..3cc7cbab5 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1003,7 +1003,7 @@ async fn vector_filter_negation() { #[actix_rt::test] async fn vector_filter_or_combination() { -let index = crate::vector::shared_index_for_fragments().await; + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 2751498bf..b2ec992ba 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1776,7 +1776,7 @@ impl Index { embedder_info.embedder_id, config.config.quantized(), ); - let embeddings = reader.item_vectors(rtxn, docid)?; // MARKER + let embeddings = reader.item_vectors(rtxn, docid)?; res.insert( config.name.to_owned(), (embeddings, embedder_info.embedding_status.must_regenerate(docid)), diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c0419997c..1afdf87e6 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -234,11 +234,9 @@ impl<'a> Filter<'a> { pub fn evaluate(&self, rtxn: &heed::RoTxn<'_>, index: &Index) -> Result { // to avoid doing this for each recursive call we're going to do it ONCE ahead of time let fields_ids_map = index.fields_ids_map(rtxn)?; - let filterable_attributes_rules = dbg!(index.filterable_attributes_rules(rtxn)?); + let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - println!("{fid:?}"); - let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 0a7ac313a..79e35366c 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -9,7 +9,6 @@ pub(super) struct VectorFilter<'a> { embedder_token: Option>, fragment_token: Option>, user_provided: bool, - // TODO: not_user_provided: bool, } #[derive(Debug)] diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index d69768d4b..e07483aff 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -966,7 +966,7 @@ impl<'a, 'i> Transform<'a, 'i> { // some user provided, remove only the ids that are not user provided let to_delete = arroy.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() - })?; // MARKER + })?; for to_delete in to_delete { arroy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; From fb73b83abe67532e496b7ac2f56aa5594ec81bdd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 12:14:34 +0200 Subject: [PATCH 11/76] Fix performance --- .../milli/src/search/facet/filter_vector.rs | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 79e35366c..a0dc52bac 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -235,19 +235,33 @@ impl<'a> VectorFilter<'a> { })?; }; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + if let Some(universe) = universe { + arroy_wrapper + .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap & universe)? + } else { + arroy_wrapper + .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } } else { + let mut universe = universe.cloned(); + if self.user_provided { + let user_provided_docsids = + embedder_info.embedding_status.user_provided_docids(); + match &mut universe { + Some(universe) => *universe &= user_provided_docsids, + None => universe = Some(user_provided_docsids.clone()), + } + } + let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents + if let Some(universe) = &universe { + stats.documents & universe + } else { + stats.documents + } }; - // FIXME: performance - if self.user_provided { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - new_docids &= user_provided_docsids; - } - docids |= new_docids; } From 9e98a25e45ab68540e2aba398a0e3eadd9fc1b9d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 15:56:09 +0200 Subject: [PATCH 12/76] Fix clippy --- crates/filter-parser/src/lib.rs | 2 -- crates/milli/src/search/facet/filter_vector.rs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index b64477170..02f338673 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -614,8 +614,6 @@ impl std::fmt::Display for Token<'_> { #[cfg(test)] pub mod tests { - use std::fmt::format; - use FilterCondition as Fc; use super::*; diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index a0dc52bac..0b9cad702 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -212,7 +212,7 @@ impl<'a> VectorFilter<'a> { embedder_config.config.quantized(), ); - let mut new_docids = if let Some(fragment_token) = &self.fragment_token { + docids |= if let Some(fragment_token) = &self.fragment_token { let fragment_name = fragment_token.value(); let Some(fragment_config) = embedder_config .fragments @@ -261,8 +261,6 @@ impl<'a> VectorFilter<'a> { stats.documents } }; - - docids |= new_docids; } if let Some(universe) = universe { From 881c37393fb79208f114c9a16696f4841096daa0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:06:27 +0200 Subject: [PATCH 13/76] Add telemetry --- .../src/routes/indexes/documents.rs | 25 +++++++++++++++++-- .../src/routes/indexes/search_analytics.rs | 6 +++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index bc5539081..173b3ecc8 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -135,6 +135,7 @@ pub struct DocumentsFetchAggregator { per_document_id: bool, // if a filter was used per_filter: bool, + with_vector_filter: bool, #[serde(rename = "vector.retrieve_vectors")] retrieve_vectors: bool, @@ -153,8 +154,17 @@ pub struct DocumentsFetchAggregator { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum DocumentFetchKind { - PerDocumentId { retrieve_vectors: bool }, - Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool, ids: usize }, + PerDocumentId { + retrieve_vectors: bool, + }, + Normal { + with_filter: bool, + with_vector_filter: bool, + limit: usize, + offset: usize, + retrieve_vectors: bool, + ids: usize, + }, } impl DocumentsFetchAggregator { @@ -174,6 +184,7 @@ impl DocumentsFetchAggregator { Self { per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }), per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), + with_vector_filter: matches!(query, DocumentFetchKind::Normal { with_vector_filter, .. } if *with_vector_filter), max_limit: limit, max_offset: offset, retrieve_vectors, @@ -193,6 +204,7 @@ impl Aggregate for DocumentsFetchAggregator { Box::new(Self { per_document_id: self.per_document_id | new.per_document_id, per_filter: self.per_filter | new.per_filter, + with_vector_filter: self.with_vector_filter | new.with_vector_filter, retrieve_vectors: self.retrieve_vectors | new.retrieve_vectors, max_limit: self.max_limit.max(new.max_limit), max_offset: self.max_offset.max(new.max_offset), @@ -276,6 +288,7 @@ pub async fn get_document( retrieve_vectors: param_retrieve_vectors.0, per_document_id: true, per_filter: false, + with_vector_filter: false, max_limit: 0, max_offset: 0, max_document_ids: 0, @@ -495,6 +508,10 @@ pub async fn documents_by_query_post( analytics.publish( DocumentsFetchAggregator:: { per_filter: body.filter.is_some(), + with_vector_filter: body + .filter + .as_ref() + .is_some_and(|f| f.to_string().contains("_vectors")), retrieve_vectors: body.retrieve_vectors, max_limit: body.limit, max_offset: body.offset, @@ -596,6 +613,10 @@ pub async fn get_documents( analytics.publish( DocumentsFetchAggregator:: { per_filter: query.filter.is_some(), + with_vector_filter: query + .filter + .as_ref() + .is_some_and(|f| f.to_string().contains("_vectors")), retrieve_vectors: query.retrieve_vectors, max_limit: query.limit, max_offset: query.offset, diff --git a/crates/meilisearch/src/routes/indexes/search_analytics.rs b/crates/meilisearch/src/routes/indexes/search_analytics.rs index 07f79eba7..6b3b7ea46 100644 --- a/crates/meilisearch/src/routes/indexes/search_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/search_analytics.rs @@ -40,6 +40,7 @@ pub struct SearchAggregator { // filter filter_with_geo_radius: bool, filter_with_geo_bounding_box: bool, + filter_on_vectors: bool, // every time a request has a filter, this field must be incremented by the number of terms it contains filter_sum_of_criteria_terms: usize, // every time a request has a filter, this field must be incremented by one @@ -163,6 +164,7 @@ impl SearchAggregator { let stringified_filters = filter.to_string(); ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); + ret.filter_on_vectors = stringified_filters.contains("_vectors"); ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); } @@ -260,6 +262,7 @@ impl Aggregate for SearchAggregator { distinct, filter_with_geo_radius, filter_with_geo_bounding_box, + filter_on_vectors, filter_sum_of_criteria_terms, filter_total_number_of_criteria, used_syntax, @@ -314,6 +317,7 @@ impl Aggregate for SearchAggregator { // filter self.filter_with_geo_radius |= filter_with_geo_radius; self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; + self.filter_on_vectors |= filter_on_vectors; self.filter_sum_of_criteria_terms = self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); self.filter_total_number_of_criteria = @@ -388,6 +392,7 @@ impl Aggregate for SearchAggregator { distinct, filter_with_geo_radius, filter_with_geo_bounding_box, + filter_on_vectors, filter_sum_of_criteria_terms, filter_total_number_of_criteria, used_syntax, @@ -445,6 +450,7 @@ impl Aggregate for SearchAggregator { "filter": { "with_geoRadius": filter_with_geo_radius, "with_geoBoundingBox": filter_with_geo_bounding_box, + "on_vectors": filter_on_vectors, "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), }, From feb53104e51a244660e3eb2d082f7b6be42d226c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:19:55 +0200 Subject: [PATCH 14/76] Grammar --- .../src/scheduler/test_document_addition.rs | 2 +- crates/meilisearch/src/routes/tasks.rs | 8 ++++---- crates/meilisearch/tests/documents/errors.rs | 2 +- crates/meilisearch/tests/search/errors.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/test_document_addition.rs b/crates/index-scheduler/src/scheduler/test_document_addition.rs index b642f5604..7ca72da95 100644 --- a/crates/index-scheduler/src/scheduler/test_document_addition.rs +++ b/crates/index-scheduler/src/scheduler/test_document_addition.rs @@ -736,7 +736,7 @@ fn test_document_addition_mixed_rights_with_index() { #[test] fn test_document_addition_mixed_right_without_index_starts_with_cant_create() { // We're going to autobatch multiple document addition. - // - The index does not exists + // - The index does not exist // - The first document addition don't have the right to create an index // - The second do. They should not batch together. // - The second should batch with everything else as it's going to create an index. diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index 95c105894..fb0f73425 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -336,7 +336,7 @@ impl Aggregate for TaskFilterAnalytics Date: Tue, 8 Jul 2025 16:23:45 +0200 Subject: [PATCH 15/76] Add test --- crates/meilisearch/tests/search/filters.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 3cc7cbab5..3b26ab4ee 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -767,6 +767,26 @@ async fn vector_filter_all_embedders() { "#); } +#[actix_rt::test] +async fn vector_filter_missing_fragment() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.\n15:24 _vectors.rest.fragments EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); +} + #[actix_rt::test] async fn vector_filter_non_existant_embedder() { let index = crate::vector::shared_index_for_fragments().await; From 8adf6141e09cc9dce662ddcc31a65f89fd81fdec Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:55:43 +0200 Subject: [PATCH 16/76] Fix old test --- crates/milli/src/test_index.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6bb6b1345..0ec348301 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -19,7 +19,9 @@ use crate::update::{ }; use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; use crate::vector::RuntimeEmbedders; -use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; +use crate::{ + db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, UserError, +}; pub(crate) struct TempIndex { pub inner: Index, @@ -1341,8 +1343,8 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap(); - assert!(results.candidates.is_empty()); + .unwrap_err(); + assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); index .update_settings(|settings| { @@ -1373,6 +1375,6 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap(); - assert!(results.candidates.is_empty()); + .unwrap_err(); + assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); } From 39f808714d6ae7667d914817dcf645c17c027374 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 9 Jul 2025 18:03:32 +0200 Subject: [PATCH 17/76] Implement a documentTemplate filter --- crates/meilisearch/tests/search/filters.rs | 79 +++++++++++++++++++ crates/meilisearch/tests/vector/mod.rs | 4 +- crates/meilisearch/tests/vector/rest.rs | 2 +- .../milli/src/search/facet/filter_vector.rs | 39 +++++++-- 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 3b26ab4ee..d0f388220 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -989,6 +989,85 @@ async fn vector_filter_specific_fragment_user_provided() { "#); } +#[actix_rt::test] +async fn vector_filter_document_template_but_fragments_used() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.documentTemplate EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); +} + +#[actix_rt::test] +async fn vector_filter_document_template() { + let (_mock, setting) = crate::vector::create_mock().await; + let server = crate::vector::get_server_vector().await; + let index = server.index("doggo"); + + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": setting, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel"}, + {"id": 3, "name": "iko" } + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + index.wait_task(value.uid()).await.succeeded(); + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.documentTemplate EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "echo" + }, + { + "name": "intel" + }, + { + "name": "iko" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); +} + #[actix_rt::test] async fn vector_filter_negation() { let index = crate::vector::shared_index_for_fragments().await; diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 9ba37cae3..8851d029e 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,9 +14,9 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; -pub use fragments::shared_index_for_fragments; +pub use {fragments::shared_index_for_fragments, rest::create_mock}; -async fn get_server_vector() -> Server { +pub async fn get_server_vector() -> Server { Server::new().await } diff --git a/crates/meilisearch/tests/vector/rest.rs b/crates/meilisearch/tests/vector/rest.rs index 974341cd0..dae9e9139 100644 --- a/crates/meilisearch/tests/vector/rest.rs +++ b/crates/meilisearch/tests/vector/rest.rs @@ -12,7 +12,7 @@ use crate::common::Value; use crate::json; use crate::vector::{get_server_vector, GetAllDocumentsOptions}; -async fn create_mock() -> (&'static MockServer, Value) { +pub async fn create_mock() -> (&'static MockServer, Value) { let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 0b9cad702..e3ec698f5 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -8,6 +8,7 @@ use crate::Index; pub(super) struct VectorFilter<'a> { embedder_token: Option>, fragment_token: Option>, + document_template: bool, user_provided: bool, } @@ -17,6 +18,7 @@ pub enum VectorFilterError<'a> { InvalidPrefix(Token<'a>), MissingFragmentName(Token<'a>), UserProvidedWithFragment(Token<'a>), + DocumentTemplateWithFragment(Token<'a>), LeftoverToken(Token<'a>), EmbedderDoesNotExist { embedder: &'a Token<'a>, @@ -52,6 +54,9 @@ impl std::fmt::Display for VectorFilterError<'_> { UserProvidedWithFragment(_token) => { write!(f, "Vector filter cannot specify both a fragment name and userProvided.") } + DocumentTemplateWithFragment(_token) => { + write!(f, "Vector filter cannot specify both a fragment name and documentTemplate.") + } LeftoverToken(token) => { write!(f, "Vector filter has leftover token: `{}`.", token.value()) } @@ -105,6 +110,7 @@ impl<'a> From> for Error { InvalidPrefix(token) | MissingFragmentName(token) | UserProvidedWithFragment(token) + | DocumentTemplateWithFragment(token) | LeftoverToken(token) => token.clone().as_external_error(err).into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), @@ -123,6 +129,8 @@ impl<'a> VectorFilter<'a> { /// - `_vectors` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.documentTemplate` + /// - `_vectors.{embedder_name}.documentTemplate.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -149,10 +157,22 @@ impl<'a> VectorFilter<'a> { user_provided_token = split.next(); } + let mut document_template_token = None; + if split.peek().map(|t| t.value()) == Some("documentTemplate") + || split.peek().map(|t| t.value()) == Some("document_template") + { + document_template_token = split.next(); + } + if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { return Err(UserProvidedWithFragment(user_provided_token.clone()))?; } + if let (Some(_), Some(document_template_token)) = (&fragment_name, &document_template_token) + { + return Err(DocumentTemplateWithFragment(document_template_token.clone()))?; + } + if let Some(next) = split.next() { return Err(LeftoverToken(next))?; } @@ -161,6 +181,7 @@ impl<'a> VectorFilter<'a> { embedder_token: embedder_name, fragment_token: fragment_name, user_provided: user_provided_token.is_some(), + document_template: document_template_token.is_some(), }) } @@ -176,7 +197,8 @@ impl<'a> VectorFilter<'a> { let mut embedders = Vec::new(); if let Some(embedder_token) = &self.embedder_token { let embedder_name = embedder_token.value(); - let Some(embedder_config) = + + let Some(embedding_config) = embedding_configs.iter().find(|config| config.name == embedder_name) else { return Err(EmbedderDoesNotExist { @@ -184,6 +206,7 @@ impl<'a> VectorFilter<'a> { available: embedding_configs.iter().map(|c| c.name.clone()).collect(), })?; }; + let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? else { return Err(EmbedderDoesNotExist { @@ -192,7 +215,11 @@ impl<'a> VectorFilter<'a> { })?; }; - embedders.push((embedder_config, embedder_info)); + if self.document_template && !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + embedders.push((embedding_config, embedder_info)); } else { for embedder_config in embedding_configs.iter() { let Some(embedder_info) = @@ -205,16 +232,16 @@ impl<'a> VectorFilter<'a> { }; let mut docids = RoaringBitmap::new(); - for (embedder_config, embedder_info) in embedders { + for (embedding_config, embedder_info) in embedders { let arroy_wrapper = ArroyWrapper::new( index.vector_arroy, embedder_info.embedder_id, - embedder_config.config.quantized(), + embedding_config.config.quantized(), ); docids |= if let Some(fragment_token) = &self.fragment_token { let fragment_name = fragment_token.value(); - let Some(fragment_config) = embedder_config + let Some(fragment_config) = embedding_config .fragments .as_slice() .iter() @@ -226,7 +253,7 @@ impl<'a> VectorFilter<'a> { .as_ref() .expect("there can't be a fragment without an embedder"), fragment: fragment_token, - available: embedder_config + available: embedding_config .fragments .as_slice() .iter() From a3b8c2b71fc1aa2766a94b6daaf7c3872c466d01 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 9 Jul 2025 18:21:52 +0200 Subject: [PATCH 18/76] Gate behind multimodal experimental feature --- crates/filter-parser/src/lib.rs | 19 +++++++++++++++++++ crates/meilisearch/src/search/mod.rs | 14 +++++++++++++- crates/meilisearch/tests/search/filters.rs | 20 ++++++++++++++++++++ crates/milli/src/search/facet/filter.rs | 4 ++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 02f338673..1590b08fd 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -189,6 +189,25 @@ impl<'a> FilterCondition<'a> { } } + pub fn use_vector_filter(&self) -> Option<&Token> { + match self { + FilterCondition::Condition { fid, op: _ } => { + if fid.value().starts_with("_vectors.") || fid.value() == "_vectors" { + Some(fid) + } else { + None + } + } + FilterCondition::Not(this) => this.use_vector_filter(), + FilterCondition::Or(seq) | FilterCondition::And(seq) => { + seq.iter().find_map(|filter| filter.use_vector_filter()) + } + FilterCondition::GeoLowerThan { .. } + | FilterCondition::GeoBoundingBox { .. } + | FilterCondition::In { .. } => None, + } + } + pub fn fids(&self, depth: usize) -> Box + '_> { if depth == 0 { return Box::new(std::iter::empty()); diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1c987a70c..e82f4dff7 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -2077,7 +2077,7 @@ pub(crate) fn parse_filter( })?; if let Some(ref filter) = filter { - // If the contains operator is used while the contains filter features is not enabled, errors out + // If the contains operator is used while the contains filter feature is not enabled, errors out if let Some((token, error)) = filter.use_contains_operator().zip(features.check_contains_filter().err()) { @@ -2088,6 +2088,18 @@ pub(crate) fn parse_filter( } } + if let Some(ref filter) = filter { + // If a vector filter is used while the multi modal feature is not enabled, errors out + if let Some((token, error)) = + filter.use_vector_filter().zip(features.check_multimodal("using a vector filter").err()) + { + return Err(ResponseError::from_msg( + token.as_external_error(error).to_string(), + Code::FeatureNotEnabled, + )); + } + } + Ok(filter) } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index d0f388220..ff6a0cb17 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1068,6 +1068,26 @@ async fn vector_filter_document_template() { "#); } +#[actix_rt::test] +async fn vector_filter_feature_gate() { + let index = shared_index_with_documents().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "using a vector filter requires enabling the `multimodal` experimental feature. See https://github.com/orgs/meilisearch/discussions/846\n1:9 _vectors EXISTS", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "#); +} + #[actix_rt::test] async fn vector_filter_negation() { let index = crate::vector::shared_index_for_fragments().await; diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 1afdf87e6..21a552965 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -228,6 +228,10 @@ impl<'a> Filter<'a> { pub fn use_contains_operator(&self) -> Option<&Token> { self.condition.use_contains_operator() } + + pub fn use_vector_filter(&self) -> Option<&Token> { + self.condition.use_vector_filter() + } } impl<'a> Filter<'a> { From a9309774602667c47a63ab04684c72bb73376338 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 09:37:58 +0200 Subject: [PATCH 19/76] Fix test --- crates/meilisearch/tests/search/filters.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index ff6a0cb17..aa2b06e76 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1017,6 +1017,9 @@ async fn vector_filter_document_template() { let server = crate::vector::get_server_vector().await; let index = server.index("doggo"); + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + let (response, code) = index .update_settings(json!({ "embedders": { From 30fd546c1227bbfc3e71e3541f41847fca4c4713 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 16:43:10 +0200 Subject: [PATCH 20/76] Format --- crates/milli/src/vector/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f64223e41..3d77e78d7 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -485,6 +485,8 @@ impl ArroyWrapper { limit: usize, filter: Option<&RoaringBitmap>, ) -> Result, arroy::Error> { + println!("nns_by_vector: quantized={} {:?} limit={} filter={:?}", + self.quantized, vector, limit, filter); if self.quantized { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { @@ -517,6 +519,8 @@ impl ArroyWrapper { results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + println!("nns_by_vector: results={:?}", results); + Ok(results) } From f244439b4f7493edb4a7f2387f80ae932cd3229b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 16:43:45 +0200 Subject: [PATCH 21/76] Revert "Format" This reverts commit 30fd546c1227bbfc3e71e3541f41847fca4c4713. --- crates/milli/src/vector/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 3d77e78d7..f64223e41 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -485,8 +485,6 @@ impl ArroyWrapper { limit: usize, filter: Option<&RoaringBitmap>, ) -> Result, arroy::Error> { - println!("nns_by_vector: quantized={} {:?} limit={} filter={:?}", - self.quantized, vector, limit, filter); if self.quantized { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { @@ -519,8 +517,6 @@ impl ArroyWrapper { results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - println!("nns_by_vector: results={:?}", results); - Ok(results) } From ab07e9480ec8364dc2555252e5bc8b32c02d7379 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 21 Jul 2025 18:22:10 +0200 Subject: [PATCH 22/76] Resolve post-merge issues --- .../src/routes/indexes/documents.rs | 2 +- crates/meilisearch/tests/search/filters.rs | 28 +++++++++---------- crates/meilisearch/tests/vector/mod.rs | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 0b70ad86f..cf6181de5 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -139,7 +139,7 @@ pub struct DocumentsFetchAggregator { // if a filter was used per_filter: bool, with_vector_filter: bool, - + // if documents were sorted sort: bool, diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index aa2b06e76..fd5bc57db 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -4,8 +4,8 @@ use tempfile::TempDir; use super::test_settings_documents_indexing_swapping_and_search; use crate::common::{ - default_settings, shared_index_with_documents, shared_index_with_nested_documents, Server, - DOCUMENTS, NESTED_DOCUMENTS, + default_settings, shared_index_for_fragments, shared_index_with_documents, + shared_index_with_nested_documents, Server, DOCUMENTS, NESTED_DOCUMENTS, }; use crate::json; @@ -734,7 +734,7 @@ async fn test_filterable_attributes_priority() { #[actix_rt::test] async fn vector_filter_all_embedders() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -769,7 +769,7 @@ async fn vector_filter_all_embedders() { #[actix_rt::test] async fn vector_filter_missing_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -789,7 +789,7 @@ async fn vector_filter_missing_fragment() { #[actix_rt::test] async fn vector_filter_non_existant_embedder() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -809,7 +809,7 @@ async fn vector_filter_non_existant_embedder() { #[actix_rt::test] async fn vector_filter_all_embedders_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name @@ -831,7 +831,7 @@ async fn vector_filter_all_embedders_user_provided() { #[actix_rt::test] async fn vector_filter_specific_embedder() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -866,7 +866,7 @@ async fn vector_filter_specific_embedder() { #[actix_rt::test] async fn vector_filter_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -892,7 +892,7 @@ async fn vector_filter_user_provided() { #[actix_rt::test] async fn vector_filter_specific_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -951,7 +951,7 @@ async fn vector_filter_specific_fragment() { #[actix_rt::test] async fn vector_filter_non_existant_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -971,7 +971,7 @@ async fn vector_filter_non_existant_fragment() { #[actix_rt::test] async fn vector_filter_specific_fragment_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -991,7 +991,7 @@ async fn vector_filter_specific_fragment_user_provided() { #[actix_rt::test] async fn vector_filter_document_template_but_fragments_used() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -1093,7 +1093,7 @@ async fn vector_filter_feature_gate() { #[actix_rt::test] async fn vector_filter_negation() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -1125,7 +1125,7 @@ async fn vector_filter_negation() { #[actix_rt::test] async fn vector_filter_or_combination() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 8851d029e..8a701ac83 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,7 +14,7 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; -pub use {fragments::shared_index_for_fragments, rest::create_mock}; +pub use rest::create_mock; pub async fn get_server_vector() -> Server { Server::new().await From 0014ed31145b1c9a4021ab6069362cc23c884b37 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 10:56:05 +0200 Subject: [PATCH 23/76] Apply review suggestions --- crates/meilisearch/tests/search/filters.rs | 21 +- .../milli/src/search/facet/filter_vector.rs | 278 ++++++++++-------- crates/milli/src/vector/db.rs | 1 + 3 files changed, 161 insertions(+), 139 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index fd5bc57db..4701e986d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot specify both a fragment name and userProvided.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "Index `[uuid]`: Vector filter cannot have both `other` and `userProvided`.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1022,19 +1022,19 @@ async fn vector_filter_document_template() { let (response, code) = index .update_settings(json!({ - "embedders": { - "rest": setting, - }, + "embedders": { + "rest": setting, + }, })) .await; snapshot!(code, @"202 Accepted"); server.wait_task(response.uid()).await.succeeded(); let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel"}, - {"id": 3, "name": "iko" } + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel"}, + {"id": 3, "name": "iko" } ]); let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); @@ -1052,9 +1052,6 @@ async fn vector_filter_document_template() { { "name": "kefir" }, - { - "name": "echo" - }, { "name": "intel" }, @@ -1066,7 +1063,7 @@ async fn vector_filter_document_template() { "processingTimeMs": "[duration]", "limit": 20, "offset": 0, - "estimatedTotalHits": 4 + "estimatedTotalHits": 3 } "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index e3ec698f5..3a0f86637 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -2,14 +2,107 @@ use filter_parser::Token; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; +use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; +#[derive(Debug)] +enum VectorFilterInner<'a> { + Fragment { embedder_token: Token<'a>, fragment_token: Token<'a> }, + DocumentTemplate { embedder_token: Token<'a> }, + UserProvided { embedder_token: Token<'a> }, + FullEmbedder { embedder_token: Token<'a> }, +} + +impl VectorFilterInner<'_> { + fn evaluate_inner( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedding_configs: &[IndexEmbeddingConfig], + regenerate: bool, + ) -> crate::Result { + let embedder = match self { + VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, + VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, + VectorFilterInner::UserProvided { embedder_token } => embedder_token, + VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, + }; + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); + + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); + + let mut docids = match self { + VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilterInner::DocumentTemplate { .. } => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilterInner::UserProvided { .. } => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilterInner::FullEmbedder { .. } => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; + + if regenerate { + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + docids &= skip_regenerate; + } + + Ok(docids) + } +} + +#[derive(Debug)] pub(super) struct VectorFilter<'a> { - embedder_token: Option>, - fragment_token: Option>, - document_template: bool, - user_provided: bool, + inner: Option>, + regenerate: bool, } #[derive(Debug)] @@ -17,8 +110,7 @@ pub enum VectorFilterError<'a> { EmptyFilter, InvalidPrefix(Token<'a>), MissingFragmentName(Token<'a>), - UserProvidedWithFragment(Token<'a>), - DocumentTemplateWithFragment(Token<'a>), + ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), LeftoverToken(Token<'a>), EmbedderDoesNotExist { embedder: &'a Token<'a>, @@ -51,11 +143,13 @@ impl std::fmt::Display for VectorFilterError<'_> { MissingFragmentName(_token) => { write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") } - UserProvidedWithFragment(_token) => { - write!(f, "Vector filter cannot specify both a fragment name and userProvided.") - } - DocumentTemplateWithFragment(_token) => { - write!(f, "Vector filter cannot specify both a fragment name and documentTemplate.") + ExclusiveOptions(tokens) => { + write!( + f, + "Vector filter cannot have both `{}` and `{}`.", + tokens.0.value(), + tokens.1.value() + ) } LeftoverToken(token) => { write!(f, "Vector filter has leftover token: `{}`.", token.value()) @@ -107,11 +201,10 @@ impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), - InvalidPrefix(token) - | MissingFragmentName(token) - | UserProvidedWithFragment(token) - | DocumentTemplateWithFragment(token) - | LeftoverToken(token) => token.clone().as_external_error(err).into(), + InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { + token.clone().as_external_error(err).into() + } + ExclusiveOptions(tokens) => tokens.1.clone().as_external_error(err).into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } @@ -127,11 +220,15 @@ impl<'a> VectorFilter<'a> { /// /// Valid formats: /// - `_vectors` + /// - `_vectors.mustRegenerate` /// - `_vectors.{embedder_name}` + /// - `_vectors.{embedder_name}.mustRegenerate` /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.userProvided.mustRegenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.userProvided` + /// - `_vectors.{embedder_name}.documentTemplate.mustRegenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.mustRegenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -151,38 +248,46 @@ impl<'a> VectorFilter<'a> { } let mut user_provided_token = None; - if split.peek().map(|t| t.value()) == Some("userProvided") - || split.peek().map(|t| t.value()) == Some("user_provided") - { + if split.peek().map(|t| t.value()) == Some("userProvided") { user_provided_token = split.next(); } let mut document_template_token = None; - if split.peek().map(|t| t.value()) == Some("documentTemplate") - || split.peek().map(|t| t.value()) == Some("document_template") - { + if split.peek().map(|t| t.value()) == Some("documentTemplate") { document_template_token = split.next(); } - if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { - return Err(UserProvidedWithFragment(user_provided_token.clone()))?; + let mut regenerate_token = None; + if split.peek().map(|t| t.value()) == Some("regenerate") { + regenerate_token = split.next(); } - if let (Some(_), Some(document_template_token)) = (&fragment_name, &document_template_token) - { - return Err(DocumentTemplateWithFragment(document_template_token.clone()))?; - } + let inner = match (fragment_name, user_provided_token, document_template_token) { + (Some(fragment_name), None, None) => Some(VectorFilterInner::Fragment { + embedder_token: embedder_name + .expect("embedder name comes before fragment so it's always Some"), + fragment_token: fragment_name, + }), + (None, Some(_), None) => Some(VectorFilterInner::UserProvided { + embedder_token: embedder_name + .expect("embedder name comes before userProvided so it's always Some"), + }), + (None, None, Some(_)) => Some(VectorFilterInner::DocumentTemplate { + embedder_token: embedder_name + .expect("embedder name comes before documentTemplate so it's always Some"), + }), + (Some(a), Some(b), _) | (_, Some(a), Some(b)) | (Some(a), None, Some(b)) => { + return Err(ExclusiveOptions(Box::new((a, b)))); + } + (None, None, None) => embedder_name + .map(|embedder_token| VectorFilterInner::FullEmbedder { embedder_token }), + }; if let Some(next) = split.next() { return Err(LeftoverToken(next))?; } - Ok(Self { - embedder_token: embedder_name, - fragment_token: fragment_name, - user_provided: user_provided_token.is_some(), - document_template: document_template_token.is_some(), - }) + Ok(Self { inner, regenerate: regenerate_token.is_some() }) } pub(super) fn evaluate( @@ -194,100 +299,19 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let mut embedders = Vec::new(); - if let Some(embedder_token) = &self.embedder_token { - let embedder_name = embedder_token.value(); - - let Some(embedding_config) = - embedding_configs.iter().find(|config| config.name == embedder_name) - else { - return Err(EmbedderDoesNotExist { - embedder: embedder_token, - available: embedding_configs.iter().map(|c| c.name.clone()).collect(), - })?; - }; - - let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? - else { - return Err(EmbedderDoesNotExist { - embedder: embedder_token, - available: embedding_configs.iter().map(|c| c.name.clone()).collect(), - })?; - }; - - if self.document_template && !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - - embedders.push((embedding_config, embedder_info)); - } else { - for embedder_config in embedding_configs.iter() { - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, &embedder_config.name)? - else { - continue; - }; - embedders.push((embedder_config, embedder_info)); - } - }; + let inners = dbg!(match self.inner { + Some(inner) => vec![inner], + None => embedding_configs + .iter() + .map(|config| VectorFilterInner::FullEmbedder { + embedder_token: Token::from(config.name.as_str()), + }) + .collect(), + }); let mut docids = RoaringBitmap::new(); - for (embedding_config, embedder_info) in embedders { - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); - - docids |= if let Some(fragment_token) = &self.fragment_token { - let fragment_name = fragment_token.value(); - let Some(fragment_config) = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - else { - return Err(FragmentDoesNotExist { - embedder: self - .embedder_token - .as_ref() - .expect("there can't be a fragment without an embedder"), - fragment: fragment_token, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; - }; - - if let Some(universe) = universe { - arroy_wrapper - .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap & universe)? - } else { - arroy_wrapper - .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? - } - } else { - let mut universe = universe.cloned(); - if self.user_provided { - let user_provided_docsids = - embedder_info.embedding_status.user_provided_docids(); - match &mut universe { - Some(universe) => *universe &= user_provided_docsids, - None => universe = Some(user_provided_docsids.clone()), - } - } - - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - if let Some(universe) = &universe { - stats.documents & universe - } else { - stats.documents - } - }; + for inner in inners.iter() { + docids |= inner.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)?; } if let Some(universe) = universe { diff --git a/crates/milli/src/vector/db.rs b/crates/milli/src/vector/db.rs index 2fea75d68..d445b47c0 100644 --- a/crates/milli/src/vector/db.rs +++ b/crates/milli/src/vector/db.rs @@ -128,6 +128,7 @@ impl EmbeddingStatus { pub fn is_user_provided(&self, docid: DocumentId) -> bool { self.user_provided.contains(docid) } + /// Whether vectors should be regenerated for that document and that embedder. pub fn must_regenerate(&self, docid: DocumentId) -> bool { let invert = self.skip_regenerate_different_from_user_provided.contains(docid); From 982e9898861f4e29b0409cf3a51e610ba308cdc5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:10:05 +0200 Subject: [PATCH 24/76] Test regenerate filter --- crates/meilisearch/tests/search/filters.rs | 36 +++++++++++++++++++ .../milli/src/search/facet/filter_vector.rs | 11 +++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 4701e986d..1a27bdf99 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1151,3 +1151,39 @@ async fn vector_filter_or_combination() { } "#); } + + +#[actix_rt::test] +async fn vector_filter_regenerate() { + let index = shared_index_for_fragments().await; + + for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { + let (value, _code) = index + .search_post(json!({ + "filter": format!("{selector} EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); + } +} + diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 3a0f86637..7fbd9c916 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -92,7 +92,7 @@ impl VectorFilterInner<'_> { if regenerate { let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids &= skip_regenerate; + docids -= skip_regenerate; } Ok(docids) @@ -220,15 +220,14 @@ impl<'a> VectorFilter<'a> { /// /// Valid formats: /// - `_vectors` - /// - `_vectors.mustRegenerate` /// - `_vectors.{embedder_name}` - /// - `_vectors.{embedder_name}.mustRegenerate` + /// - `_vectors.{embedder_name}.regenerate` /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.userProvided.mustRegenerate` + /// - `_vectors.{embedder_name}.userProvided.regenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.mustRegenerate` + /// - `_vectors.{embedder_name}.documentTemplate.regenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.mustRegenerate` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.regenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); From 6d93b36279c75901a637fe8a4aaf69461274286e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:18:41 +0200 Subject: [PATCH 25/76] Format --- crates/meilisearch/tests/search/filters.rs | 14 +- .../milli/src/search/facet/filter_vector.rs | 278 ++++++++---------- 2 files changed, 122 insertions(+), 170 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 1a27bdf99..12bfbe2ea 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1152,19 +1152,18 @@ async fn vector_filter_or_combination() { "#); } - #[actix_rt::test] async fn vector_filter_regenerate() { let index = shared_index_for_fragments().await; for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { let (value, _code) = index - .search_post(json!({ - "filter": format!("{selector} EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" + .search_post(json!({ + "filter": format!("{selector} EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" { "hits": [ { @@ -1186,4 +1185,3 @@ async fn vector_filter_regenerate() { "#); } } - diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 7fbd9c916..a59bbb5f9 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -14,108 +14,49 @@ enum VectorFilterInner<'a> { FullEmbedder { embedder_token: Token<'a> }, } -impl VectorFilterInner<'_> { - fn evaluate_inner( - &self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - embedding_configs: &[IndexEmbeddingConfig], - regenerate: bool, - ) -> crate::Result { - let embedder = match self { - VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, - VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, - VectorFilterInner::UserProvided { embedder_token } => embedder_token, - VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, - }; - let embedder_name = embedder.value(); - let available_embedders = - || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); - - let embedding_config = embedding_configs - .iter() - .find(|config| config.name == embedder_name) - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - - let embedder_info = index - .embedding_configs() - .embedder_info(rtxn, embedder_name)? - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); - - let mut docids = match self { - VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { - let fragment_name = fragment.value(); - let fragment_config = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - .ok_or_else(|| FragmentDoesNotExist { - embedder, - fragment, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; - - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? - } - VectorFilterInner::DocumentTemplate { .. } => { - if !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() - } - VectorFilterInner::UserProvided { .. } => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() - } - VectorFilterInner::FullEmbedder { .. } => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - } - }; - - if regenerate { - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids -= skip_regenerate; - } - - Ok(docids) - } -} - #[derive(Debug)] pub(super) struct VectorFilter<'a> { inner: Option>, regenerate: bool, } -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum VectorFilterError<'a> { + #[error("Vector filter cannot be empty.")] EmptyFilter, + + #[error("Vector filter must start with `_vectors` but found `{}`.", _0.value())] InvalidPrefix(Token<'a>), + + #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] MissingFragmentName(Token<'a>), + + #[error("Vector filter cannot have both `{}` and `{}`.", _0.0.value(), _0.1.value())] ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), + + #[error("Vector filter has leftover token: `{}`.", _0.value())] LeftoverToken(Token<'a>), - EmbedderDoesNotExist { - embedder: &'a Token<'a>, - available: Vec, - }, + + #[error("The embedder `{}` does not exist. {}", embedder.value(), { + if available.is_empty() { + String::from("This index does not have any configured embedders.") + } else { + let mut available = available.clone(); + available.sort_unstable(); + format!("Available embedders are: {}.", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) + } + })] + EmbedderDoesNotExist { embedder: &'a Token<'a>, available: Vec }, + + #[error("The fragment `{}` does not exist on embedder `{}`. {}", fragment.value(), embedder.value(), { + if available.is_empty() { + String::from("This embedder does not have any configured fragments.") + } else { + let mut available = available.clone(); + available.sort_unstable(); + format!("Available fragments on this embedder are: {}.", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) + } + })] FragmentDoesNotExist { embedder: &'a Token<'a>, fragment: &'a Token<'a>, @@ -125,78 +66,6 @@ pub enum VectorFilterError<'a> { use VectorFilterError::*; -impl std::error::Error for VectorFilterError<'_> {} - -impl std::fmt::Display for VectorFilterError<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - EmptyFilter => { - write!(f, "Vector filter cannot be empty.") - } - InvalidPrefix(prefix) => { - write!( - f, - "Vector filter must start with `_vectors` but found `{}`.", - prefix.value() - ) - } - MissingFragmentName(_token) => { - write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") - } - ExclusiveOptions(tokens) => { - write!( - f, - "Vector filter cannot have both `{}` and `{}`.", - tokens.0.value(), - tokens.1.value() - ) - } - LeftoverToken(token) => { - write!(f, "Vector filter has leftover token: `{}`.", token.value()) - } - EmbedderDoesNotExist { embedder, available } => { - write!(f, "The embedder `{}` does not exist.", embedder.value())?; - if available.is_empty() { - write!(f, " This index does not have configured embedders.") - } else { - write!(f, " Available embedders are: ")?; - let mut available = available.clone(); - available.sort_unstable(); - for (idx, embedder) in available.iter().enumerate() { - write!(f, "`{embedder}`")?; - if idx != available.len() - 1 { - write!(f, ", ")?; - } - } - write!(f, ".") - } - } - FragmentDoesNotExist { embedder, fragment, available } => { - write!( - f, - "The fragment `{}` does not exist on embedder `{}`.", - fragment.value(), - embedder.value(), - )?; - if available.is_empty() { - write!(f, " This embedder does not have configured fragments.") - } else { - write!(f, " Available fragments on this embedder are: ")?; - let mut available = available.clone(); - available.sort_unstable(); - for (idx, fragment) in available.iter().enumerate() { - write!(f, "`{fragment}`")?; - if idx != available.len() - 1 { - write!(f, ", ")?; - } - } - write!(f, ".") - } - } - } - } -} - impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { @@ -320,3 +189,88 @@ impl<'a> VectorFilter<'a> { Ok(docids) } } + +impl VectorFilterInner<'_> { + fn evaluate_inner( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedding_configs: &[IndexEmbeddingConfig], + regenerate: bool, + ) -> crate::Result { + let embedder = match self { + VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, + VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, + VectorFilterInner::UserProvided { embedder_token } => embedder_token, + VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, + }; + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); + + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); + + let mut docids = match self { + VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilterInner::DocumentTemplate { .. } => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilterInner::UserProvided { .. } => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilterInner::FullEmbedder { .. } => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; + + if regenerate { + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + docids -= skip_regenerate; + } + + Ok(docids) + } +} From 3362fb8476860efa36fb6a1b9fdf296e47b42d11 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:21:06 +0200 Subject: [PATCH 26/76] Remove print --- crates/milli/src/search/facet/filter_vector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index a59bbb5f9..e2c00a16a 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -167,7 +167,7 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let inners = dbg!(match self.inner { + let inners = match self.inner { Some(inner) => vec![inner], None => embedding_configs .iter() @@ -175,7 +175,7 @@ impl<'a> VectorFilter<'a> { embedder_token: Token::from(config.name.as_str()), }) .collect(), - }); + }; let mut docids = RoaringBitmap::new(); for inner in inners.iter() { From 776e55d2096febba707582273243721eebb7b1dc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:37:21 +0200 Subject: [PATCH 27/76] Improve code readability --- crates/milli/src/search/facet/filter_vector.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index e2c00a16a..83b3adcc3 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,5 +1,5 @@ use filter_parser::Token; -use roaring::RoaringBitmap; +use roaring::{MultiOps, RoaringBitmap}; use crate::error::{Error, UserError}; use crate::vector::db::IndexEmbeddingConfig; @@ -177,10 +177,10 @@ impl<'a> VectorFilter<'a> { .collect(), }; - let mut docids = RoaringBitmap::new(); - for inner in inners.iter() { - docids |= inner.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)?; - } + let mut docids = inners + .iter() + .map(|i| i.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)) + .union()?; if let Some(universe) = universe { docids &= universe; From aa5a1f333aa01a3161fa979cec4a85da35c5b803 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 23 Jul 2025 15:33:17 +0200 Subject: [PATCH 28/76] Refactor to support less combinations --- crates/meilisearch/tests/search/filters.rs | 69 +++++---- .../milli/src/search/facet/filter_vector.rs | 140 +++++++++--------- 2 files changed, 109 insertions(+), 100 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 12bfbe2ea..cd2da747d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot have both `other` and `userProvided`.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `userProvided`.\n15:24 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1156,32 +1156,45 @@ async fn vector_filter_or_combination() { async fn vector_filter_regenerate() { let index = shared_index_for_fragments().await; - for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { - let (value, _code) = index - .search_post(json!({ - "filter": format!("{selector} EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "hits": [ - { - "name": "kefir" - }, - { - "name": "intel" - }, - { - "name": "dustin" - } - ], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 3 - } - "#); + let (value, _code) = index + .search_post(json!({ + "filter": format!("_vectors.rest.regenerate EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 } + "#); + + let (value, _code) = index + .search_post(json!({ + "filter": format!("_vectors.rest.fragments.basic.regenerate EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `regenerate`.\n15:24 _vectors.rest.fragments.basic.regenerate EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 83b3adcc3..91f138685 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -8,16 +8,17 @@ use crate::Index; #[derive(Debug)] enum VectorFilterInner<'a> { - Fragment { embedder_token: Token<'a>, fragment_token: Token<'a> }, - DocumentTemplate { embedder_token: Token<'a> }, - UserProvided { embedder_token: Token<'a> }, - FullEmbedder { embedder_token: Token<'a> }, + Fragment(Token<'a>), + DocumentTemplate, + UserProvided, + Regenerate, + None, } #[derive(Debug)] pub(super) struct VectorFilter<'a> { - inner: Option>, - regenerate: bool, + embedder: Option>, + inner: VectorFilterInner<'a>, } #[derive(Debug, thiserror::Error)] @@ -31,8 +32,10 @@ pub enum VectorFilterError<'a> { #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] MissingFragmentName(Token<'a>), - #[error("Vector filter cannot have both `{}` and `{}`.", _0.0.value(), _0.1.value())] - ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), + #[error("Vector filter cannot have both {}.", { + _0.iter().map(|t| format!("`{}`", t.value())).collect::>().join(" and ") + })] + ExclusiveOptions(Vec>), #[error("Vector filter has leftover token: `{}`.", _0.value())] LeftoverToken(Token<'a>), @@ -73,7 +76,12 @@ impl<'a> From> for Error { InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { token.clone().as_external_error(err).into() } - ExclusiveOptions(tokens) => tokens.1.clone().as_external_error(err).into(), + ExclusiveOptions(tokens) => tokens + .first() + .cloned() + .unwrap_or_else(|| Token::from("")) // Should never happen: tokens is never created empty + .as_external_error(err) + .into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } @@ -92,11 +100,8 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.regenerate` /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.userProvided.regenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.regenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.regenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -108,54 +113,53 @@ impl<'a> VectorFilter<'a> { let embedder_name = split.next(); - let mut fragment_name = None; + let mut fragment_tokens = None; if split.peek().map(|t| t.value()) == Some("fragments") { let token = split.next().expect("it was peeked before"); + let name = split.next().ok_or_else(|| MissingFragmentName(token.clone()))?; - fragment_name = Some(split.next().ok_or(MissingFragmentName(token))?); + fragment_tokens = Some((token, name)); } + let mut remaining_tokens = split.collect::>(); + let mut user_provided_token = None; - if split.peek().map(|t| t.value()) == Some("userProvided") { - user_provided_token = split.next(); + if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "userProvided") { + user_provided_token = Some(remaining_tokens.remove(position)); } let mut document_template_token = None; - if split.peek().map(|t| t.value()) == Some("documentTemplate") { - document_template_token = split.next(); + if let Some(position) = + remaining_tokens.iter().position(|t| t.value() == "documentTemplate") + { + document_template_token = Some(remaining_tokens.remove(position)); } let mut regenerate_token = None; - if split.peek().map(|t| t.value()) == Some("regenerate") { - regenerate_token = split.next(); + if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "regenerate") { + regenerate_token = Some(remaining_tokens.remove(position)); } - let inner = match (fragment_name, user_provided_token, document_template_token) { - (Some(fragment_name), None, None) => Some(VectorFilterInner::Fragment { - embedder_token: embedder_name - .expect("embedder name comes before fragment so it's always Some"), - fragment_token: fragment_name, - }), - (None, Some(_), None) => Some(VectorFilterInner::UserProvided { - embedder_token: embedder_name - .expect("embedder name comes before userProvided so it's always Some"), - }), - (None, None, Some(_)) => Some(VectorFilterInner::DocumentTemplate { - embedder_token: embedder_name - .expect("embedder name comes before documentTemplate so it's always Some"), - }), - (Some(a), Some(b), _) | (_, Some(a), Some(b)) | (Some(a), None, Some(b)) => { - return Err(ExclusiveOptions(Box::new((a, b)))); - } - (None, None, None) => embedder_name - .map(|embedder_token| VectorFilterInner::FullEmbedder { embedder_token }), - }; - - if let Some(next) = split.next() { - return Err(LeftoverToken(next))?; + if !remaining_tokens.is_empty() { + return Err(LeftoverToken(remaining_tokens.remove(0))); } - Ok(Self { inner, regenerate: regenerate_token.is_some() }) + let inner = + match (fragment_tokens, user_provided_token, document_template_token, regenerate_token) + { + (Some((_token, name)), None, None, None) => VectorFilterInner::Fragment(name), + (None, Some(_), None, None) => VectorFilterInner::UserProvided, + (None, None, Some(_), None) => VectorFilterInner::DocumentTemplate, + (None, None, None, Some(_)) => VectorFilterInner::Regenerate, + (None, None, None, None) => VectorFilterInner::None, + (a, b, c, d) => { + let a = a.map(|(token, _)| token); + let present = [a, b, c, d].into_iter().flatten().collect(); + return Err(ExclusiveOptions(present)); + } + }; + + Ok(Self { inner, embedder: embedder_name }) } pub(super) fn evaluate( @@ -167,19 +171,16 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let inners = match self.inner { - Some(inner) => vec![inner], - None => embedding_configs - .iter() - .map(|config| VectorFilterInner::FullEmbedder { - embedder_token: Token::from(config.name.as_str()), - }) - .collect(), + let embedders = match self.embedder { + Some(embedder) => vec![embedder], + None => { + embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect() + } }; - let mut docids = inners + let mut docids = embedders .iter() - .map(|i| i.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)) + .map(|e| self.inner.evaluate(rtxn, index, e, &embedding_configs)) .union()?; if let Some(universe) = universe { @@ -191,19 +192,13 @@ impl<'a> VectorFilter<'a> { } impl VectorFilterInner<'_> { - fn evaluate_inner( + fn evaluate( &self, rtxn: &heed::RoTxn<'_>, index: &Index, + embedder: &Token<'_>, embedding_configs: &[IndexEmbeddingConfig], - regenerate: bool, ) -> crate::Result { - let embedder = match self { - VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, - VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, - VectorFilterInner::UserProvided { embedder_token } => embedder_token, - VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, - }; let embedder_name = embedder.value(); let available_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); @@ -224,8 +219,8 @@ impl VectorFilterInner<'_> { embedding_config.config.quantized(), ); - let mut docids = match self { - VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let docids = match self { + VectorFilterInner::Fragment(fragment) => { let fragment_name = fragment.value(); let fragment_config = embedding_config .fragments @@ -245,7 +240,7 @@ impl VectorFilterInner<'_> { arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? } - VectorFilterInner::DocumentTemplate { .. } => { + VectorFilterInner::DocumentTemplate => { if !embedding_config.fragments.as_slice().is_empty() { return Ok(RoaringBitmap::new()); } @@ -255,22 +250,23 @@ impl VectorFilterInner<'_> { arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; stats.documents - user_provided_docsids.clone() } - VectorFilterInner::UserProvided { .. } => { + VectorFilterInner::UserProvided => { let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); user_provided_docsids.clone() } - VectorFilterInner::FullEmbedder { .. } => { + VectorFilterInner::Regenerate => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + stats.documents - skip_regenerate + } + VectorFilterInner::None => { let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; stats.documents } }; - if regenerate { - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids -= skip_regenerate; - } - Ok(docids) } } From bb4d57386280796928f311133eb14d1a4af470a4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 14:56:35 +0200 Subject: [PATCH 29/76] Switch to a nom parser --- crates/filter-parser/src/condition.rs | 60 ++++ crates/filter-parser/src/error.rs | 34 ++ crates/filter-parser/src/lib.rs | 46 ++- crates/filter-parser/src/value.rs | 33 ++ crates/meilisearch/tests/search/filters.rs | 18 +- crates/milli/src/search/facet/filter.rs | 23 +- .../milli/src/search/facet/filter_vector.rs | 306 ++++++------------ 7 files changed, 269 insertions(+), 251 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 0fc007bf1..af0767706 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -7,11 +7,20 @@ use nom::branch::alt; use nom::bytes::complete::tag; +use nom::character::complete::char; +use nom::character::complete::multispace0; use nom::character::complete::multispace1; use nom::combinator::cut; +use nom::combinator::map; +use nom::combinator::value; +use nom::sequence::preceded; use nom::sequence::{terminated, tuple}; use Condition::*; +use crate::error::IResultExt; +use crate::value::parse_vector_value; +use crate::ErrorKind; +use crate::VectorFilter; use crate::{parse_value, FilterCondition, IResult, Span, Token}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -113,6 +122,57 @@ pub fn parse_not_exists(input: Span) -> IResult { Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Exists })))) } +fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_>)> { + let (input, _) = multispace0(input)?; + let (input, fid) = tag("_vectors")(input)?; + + if let Ok((input, _)) = multispace1::<_, crate::Error>(input) { + return Ok((input, (Token::from(fid), None, VectorFilter::None))); + } + + let (input, _) = char('.')(input)?; + + // From this point, we are certain this is a vector filter, so our errors must be final. + // We could use nom's `cut`` but it's better to be explicit about the errors + + let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + + let (input, filter) = alt(( + map( + preceded(tag(".fragments"), |input| { + let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?; + parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidFragment) + }), + VectorFilter::Fragment, + ), + value(VectorFilter::UserProvided, tag(".userProvided")), + value(VectorFilter::DocumentTemplate, tag(".documentTemplate")), + value(VectorFilter::Regenerate, tag(".regenerate")), + value(VectorFilter::None, nom::combinator::success("")), + ))(input)?; + + let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; + + Ok((input, (Token::from(fid), Some(embedder_name), filter))) +} + +/// vectors_exists = vectors "EXISTS" +pub fn parse_vectors_exists(input: Span) -> IResult { + let (input, (fid, embedder, filter)) = terminated(parse_vectors, tag("EXISTS"))(input)?; + + Ok((input, FilterCondition::VectorExists { fid, embedder, filter })) +} +/// vectors_not_exists = vectors "NOT" WS+ "EXISTS" +pub fn parse_vectors_not_exists(input: Span) -> IResult { + let (input, (fid, embedder, filter)) = parse_vectors(input)?; + + let (input, _) = tuple((tag("NOT"), multispace1, tag("EXISTS")))(input)?; + Ok(( + input, + FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), + )) +} + /// contains = value "CONTAINS" value pub fn parse_contains(input: Span) -> IResult { let (input, (fid, contains, value)) = diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 855ce983e..cf2419b01 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -42,6 +42,23 @@ pub fn cut_with_err<'a, O>( } } +pub trait IResultExt<'a> { + fn map_cut(self, kind: ErrorKind<'a>) -> Self; +} + +impl<'a, T> IResultExt<'a> for IResult<'a, T> { + fn map_cut(self, kind: ErrorKind<'a>) -> Self { + self.map_err(move |e: nom::Err>| { + let input = match e { + nom::Err::Incomplete(_) => return e, + nom::Err::Error(e) => *e.context(), + nom::Err::Failure(e) => *e.context(), + }; + nom::Err::Failure(Error::new_from_kind(input, kind)) + }) + } +} + #[derive(Debug)] pub struct Error<'a> { context: Span<'a>, @@ -76,6 +93,11 @@ pub enum ErrorKind<'a> { InternalError(error::ErrorKind), DepthLimitReached, External(String), + + VectorFilterLeftover, + VectorFilterInvalidEmbedder, + VectorFilterMissingFragment, + VectorFilterInvalidFragment, } impl<'a> Error<'a> { @@ -169,6 +191,18 @@ impl Display for Error<'_> { ErrorKind::MisusedGeoBoundingBox => { writeln!(f, "The `_geoBoundingBox` filter is an operation and can't be used as a value.")? } + ErrorKind::VectorFilterLeftover => { + writeln!(f, "The vector filter has leftover tokens.")? + } + ErrorKind::VectorFilterInvalidFragment => { + writeln!(f, "The vector filter's fragment is invalid.")? + } + ErrorKind::VectorFilterMissingFragment => { + writeln!(f, "The vector filter is missing a fragment name.")? + } + ErrorKind::VectorFilterInvalidEmbedder => { + writeln!(f, "The vector filter's embedder is invalid.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 1590b08fd..b5697f914 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -65,6 +65,9 @@ use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; +use crate::condition::{parse_vectors_exists, parse_vectors_not_exists}; +use crate::error::IResultExt; + pub type Span<'a> = LocatedSpan<&'a str, &'a str>; type IResult<'a, Ret> = nom::IResult, Ret, Error<'a>>; @@ -146,6 +149,15 @@ impl<'a> From<&'a str> for Token<'a> { } } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VectorFilter<'a> { + Fragment(Token<'a>), + DocumentTemplate, + UserProvided, + Regenerate, + None, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum FilterCondition<'a> { Not(Box), @@ -153,6 +165,7 @@ pub enum FilterCondition<'a> { In { fid: Token<'a>, els: Vec> }, Or(Vec), And(Vec), + VectorExists { fid: Token<'a>, embedder: Option>, filter: VectorFilter<'a> }, GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a> }, GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] }, } @@ -183,7 +196,8 @@ impl<'a> FilterCondition<'a> { FilterCondition::Or(seq) | FilterCondition::And(seq) => { seq.iter().find_map(|filter| filter.use_contains_operator()) } - FilterCondition::GeoLowerThan { .. } + FilterCondition::VectorExists { .. } + | FilterCondition::GeoLowerThan { .. } | FilterCondition::GeoBoundingBox { .. } | FilterCondition::In { .. } => None, } @@ -191,13 +205,7 @@ impl<'a> FilterCondition<'a> { pub fn use_vector_filter(&self) -> Option<&Token> { match self { - FilterCondition::Condition { fid, op: _ } => { - if fid.value().starts_with("_vectors.") || fid.value() == "_vectors" { - Some(fid) - } else { - None - } - } + FilterCondition::Condition { .. } => None, FilterCondition::Not(this) => this.use_vector_filter(), FilterCondition::Or(seq) | FilterCondition::And(seq) => { seq.iter().find_map(|filter| filter.use_vector_filter()) @@ -205,6 +213,7 @@ impl<'a> FilterCondition<'a> { FilterCondition::GeoLowerThan { .. } | FilterCondition::GeoBoundingBox { .. } | FilterCondition::In { .. } => None, + FilterCondition::VectorExists { fid, .. } => Some(fid), } } @@ -292,10 +301,7 @@ fn parse_in_body(input: Span) -> IResult> { let (input, _) = ws(word_exact("IN"))(input)?; // everything after `IN` can be a failure - let (input, _) = - cut_with_err(tag("["), |_| Error::new_from_kind(input, ErrorKind::InOpeningBracket))( - input, - )?; + let (input, _) = tag("[")(input).map_cut(ErrorKind::InOpeningBracket)?; let (input, content) = cut(parse_value_list)(input)?; @@ -529,8 +535,7 @@ fn parse_primary(input: Span, depth: usize) -> IResult { parse_is_not_null, parse_is_empty, parse_is_not_empty, - parse_exists, - parse_not_exists, + alt((parse_vectors_exists, parse_vectors_not_exists, parse_exists, parse_not_exists)), parse_to, parse_contains, parse_not_contains, @@ -586,6 +591,19 @@ impl std::fmt::Display for FilterCondition<'_> { } write!(f, "]") } + FilterCondition::VectorExists { fid: _, embedder, filter: inner } => { + write!(f, "_vectors")?; + if let Some(embedder) = embedder { + write!(f, ".{embedder:?}")?; + } + match inner { + VectorFilter::Fragment(fragment) => write!(f, ".fragments.{fragment:?}"), + VectorFilter::DocumentTemplate => write!(f, ".documentTemplate"), + VectorFilter::UserProvided => write!(f, ".userProvided"), + VectorFilter::Regenerate => write!(f, ".regenerate"), + VectorFilter::None => Ok(()), + } + } FilterCondition::GeoLowerThan { point, radius } => { write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) } diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index 98cac39fe..345f0b0a2 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -80,6 +80,39 @@ pub fn word_exact<'a, 'b: 'a>(tag: &'b str) -> impl Fn(Span<'a>) -> IResult<'a, } } +/// vector_value = ( non_dot_word | singleQuoted | doubleQuoted) +pub fn parse_vector_value(input: Span) -> IResult { + pub fn non_dot_word(input: Span) -> IResult { + let (input, word) = take_while1(|c| is_value_component(c) && c != '.')(input)?; + Ok((input, word.into())) + } + + let (input, value) = alt(( + delimited(char('\''), cut(|input| quoted_by('\'', input)), cut(char('\''))), + delimited(char('"'), cut(|input| quoted_by('"', input)), cut(char('"'))), + non_dot_word, + ))(input)?; + + match unescaper::unescape(value.value()) { + Ok(content) => { + if content.len() != value.value().len() { + Ok((input, Token::new(value.original_span(), Some(content)))) + } else { + Ok((input, value)) + } + } + Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)), + Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::InvalidEscapedNumber, + ))), + Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::MalformedValue, + ))), + } +} + /// value = WS* ( word | singleQuoted | doubleQuoted) WS+ pub fn parse_value(input: Span) -> IResult { // to get better diagnostic message we are going to strip the left whitespaces from the input right now diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index cd2da747d..67f9ebb71 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -779,7 +779,7 @@ async fn vector_filter_missing_fragment() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.\n15:24 _vectors.rest.fragments EXISTS", + "message": "The vector filter is missing a fragment name.\n24:31 _vectors.rest.fragments EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `userProvided`.\n15:24 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "The vector filter has leftover tokens.\n30:50 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1190,11 +1190,11 @@ async fn vector_filter_regenerate() { })) .await; snapshot!(value, @r#" - { - "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `regenerate`.\n15:24 _vectors.rest.fragments.basic.regenerate EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); + { + "message": "The vector filter has leftover tokens.\n30:48 _vectors.rest.fragments.basic.regenerate EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 21a552965..4e67814d3 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -10,8 +10,8 @@ use memchr::memmem::Finder; use roaring::{MultiOps, RoaringBitmap}; use serde_json::Value; -use super::{facet_range_search, filter_vector::VectorFilter}; -use crate::constants::RESERVED_GEO_FIELD_NAME; +use super::facet_range_search; +use crate::constants::{RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME}; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::heed_codec::facet::{ @@ -230,7 +230,7 @@ impl<'a> Filter<'a> { } pub fn use_vector_filter(&self) -> Option<&Token> { - self.condition.use_vector_filter() + dbg!(self.condition.use_vector_filter()) } } @@ -241,10 +241,10 @@ impl<'a> Filter<'a> { let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - let attribute = fid.value(); + let attribute = dbg!(fid.value()); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) - || VectorFilter::matches(attribute) + || attribute == RESERVED_VECTORS_FIELD_NAME { continue; } @@ -549,16 +549,6 @@ impl<'a> Filter<'a> { } FilterCondition::Condition { fid, op } => { let value = fid.value(); - if VectorFilter::matches(value) { - if !matches!(op, Condition::Exists) { - return Err(Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter can only be used with the `exists` operator", - )))); - } - let vector_filter = VectorFilter::parse(fid)?; - return vector_filter.evaluate(rtxn, index, universe); - } - let Some(field_id) = field_ids_map.id(value) else { return Ok(RoaringBitmap::new()); }; @@ -616,6 +606,9 @@ impl<'a> Filter<'a> { Ok(RoaringBitmap::new()) } } + FilterCondition::VectorExists { fid: _, embedder, filter } => { + super::filter_vector::evaluate(rtxn, index, universe, embedder.clone(), filter) + } FilterCondition::GeoLowerThan { point, radius } => { if index.is_geo_filtering_enabled(rtxn)? { let base_point: [f64; 2] = diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 91f138685..2ddd801ed 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,45 +1,13 @@ -use filter_parser::Token; +use filter_parser::{Token, VectorFilter}; use roaring::{MultiOps, RoaringBitmap}; -use crate::error::{Error, UserError}; +use crate::error::Error; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; -#[derive(Debug)] -enum VectorFilterInner<'a> { - Fragment(Token<'a>), - DocumentTemplate, - UserProvided, - Regenerate, - None, -} - -#[derive(Debug)] -pub(super) struct VectorFilter<'a> { - embedder: Option>, - inner: VectorFilterInner<'a>, -} - #[derive(Debug, thiserror::Error)] pub enum VectorFilterError<'a> { - #[error("Vector filter cannot be empty.")] - EmptyFilter, - - #[error("Vector filter must start with `_vectors` but found `{}`.", _0.value())] - InvalidPrefix(Token<'a>), - - #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] - MissingFragmentName(Token<'a>), - - #[error("Vector filter cannot have both {}.", { - _0.iter().map(|t| format!("`{}`", t.value())).collect::>().join(" and ") - })] - ExclusiveOptions(Vec>), - - #[error("Vector filter has leftover token: `{}`.", _0.value())] - LeftoverToken(Token<'a>), - #[error("The embedder `{}` does not exist. {}", embedder.value(), { if available.is_empty() { String::from("This index does not have any configured embedders.") @@ -72,201 +40,113 @@ use VectorFilterError::*; impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { - EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), - InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { - token.clone().as_external_error(err).into() - } - ExclusiveOptions(tokens) => tokens - .first() - .cloned() - .unwrap_or_else(|| Token::from("")) // Should never happen: tokens is never created empty - .as_external_error(err) - .into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } } } -impl<'a> VectorFilter<'a> { - pub(super) fn matches(value: &str) -> bool { - value.starts_with("_vectors.") || value == "_vectors" +pub(super) fn evaluate( + rtxn: &heed::RoTxn<'_>, + index: &Index, + universe: Option<&RoaringBitmap>, + embedder: Option>, + filter: &VectorFilter<'_>, +) -> crate::Result { + let index_embedding_configs = index.embedding_configs(); + let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; + + let embedders = match embedder { + Some(embedder) => vec![embedder], + None => embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect(), + }; + + let mut docids = embedders + .iter() + .map(|e| evaluate_inner(rtxn, index, e, &embedding_configs, filter)) + .union()?; + + if let Some(universe) = universe { + docids &= universe; } - /// Parses a vector filter string. - /// - /// Valid formats: - /// - `_vectors` - /// - `_vectors.{embedder_name}` - /// - `_vectors.{embedder_name}.regenerate` - /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - pub(super) fn parse(s: &'a Token<'a>) -> Result> { - let mut split = s.split(".").peekable(); - - match split.next() { - Some(token) if token.value() == "_vectors" => (), - Some(token) => return Err(InvalidPrefix(token)), - None => return Err(EmptyFilter), - } - - let embedder_name = split.next(); - - let mut fragment_tokens = None; - if split.peek().map(|t| t.value()) == Some("fragments") { - let token = split.next().expect("it was peeked before"); - let name = split.next().ok_or_else(|| MissingFragmentName(token.clone()))?; - - fragment_tokens = Some((token, name)); - } - - let mut remaining_tokens = split.collect::>(); - - let mut user_provided_token = None; - if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "userProvided") { - user_provided_token = Some(remaining_tokens.remove(position)); - } - - let mut document_template_token = None; - if let Some(position) = - remaining_tokens.iter().position(|t| t.value() == "documentTemplate") - { - document_template_token = Some(remaining_tokens.remove(position)); - } - - let mut regenerate_token = None; - if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "regenerate") { - regenerate_token = Some(remaining_tokens.remove(position)); - } - - if !remaining_tokens.is_empty() { - return Err(LeftoverToken(remaining_tokens.remove(0))); - } - - let inner = - match (fragment_tokens, user_provided_token, document_template_token, regenerate_token) - { - (Some((_token, name)), None, None, None) => VectorFilterInner::Fragment(name), - (None, Some(_), None, None) => VectorFilterInner::UserProvided, - (None, None, Some(_), None) => VectorFilterInner::DocumentTemplate, - (None, None, None, Some(_)) => VectorFilterInner::Regenerate, - (None, None, None, None) => VectorFilterInner::None, - (a, b, c, d) => { - let a = a.map(|(token, _)| token); - let present = [a, b, c, d].into_iter().flatten().collect(); - return Err(ExclusiveOptions(present)); - } - }; - - Ok(Self { inner, embedder: embedder_name }) - } - - pub(super) fn evaluate( - self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - universe: Option<&RoaringBitmap>, - ) -> crate::Result { - let index_embedding_configs = index.embedding_configs(); - let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - - let embedders = match self.embedder { - Some(embedder) => vec![embedder], - None => { - embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect() - } - }; - - let mut docids = embedders - .iter() - .map(|e| self.inner.evaluate(rtxn, index, e, &embedding_configs)) - .union()?; - - if let Some(universe) = universe { - docids &= universe; - } - - Ok(docids) - } + Ok(docids) } -impl VectorFilterInner<'_> { - fn evaluate( - &self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - embedder: &Token<'_>, - embedding_configs: &[IndexEmbeddingConfig], - ) -> crate::Result { - let embedder_name = embedder.value(); - let available_embedders = - || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); +fn evaluate_inner( + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedder: &Token<'_>, + embedding_configs: &[IndexEmbeddingConfig], + filter: &VectorFilter<'_>, +) -> crate::Result { + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); - let embedding_config = embedding_configs - .iter() - .find(|config| config.name == embedder_name) - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - let embedder_info = index - .embedding_configs() - .embedder_info(rtxn, embedder_name)? - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); - let docids = match self { - VectorFilterInner::Fragment(fragment) => { - let fragment_name = fragment.value(); - let fragment_config = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - .ok_or_else(|| FragmentDoesNotExist { - embedder, - fragment, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; + let docids = match filter { + VectorFilter::Fragment(fragment) => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilter::DocumentTemplate => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); } - VectorFilterInner::DocumentTemplate => { - if !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() - } - VectorFilterInner::UserProvided => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() - } - VectorFilterInner::Regenerate => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - stats.documents - skip_regenerate - } - VectorFilterInner::None => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - } - }; + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilter::UserProvided => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilter::Regenerate => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + stats.documents - skip_regenerate + } + VectorFilter::None => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; - Ok(docids) - } + Ok(docids) } From 8f1b697b911c41a86ba91a1f8119690dd8ecadae Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 14:57:06 +0200 Subject: [PATCH 30/76] Format --- crates/filter-parser/src/condition.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index af0767706..1e8deef64 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -135,7 +135,8 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // From this point, we are certain this is a vector filter, so our errors must be final. // We could use nom's `cut`` but it's better to be explicit about the errors - let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + let (input, embedder_name) = + parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; let (input, filter) = alt(( map( From ad068286856e5360952863d26066e6cac1c63e86 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:24:42 +0200 Subject: [PATCH 31/76] Add tests on parser --- crates/filter-parser/src/lib.rs | 83 ++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index b5697f914..1e342d8d2 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -594,15 +594,18 @@ impl std::fmt::Display for FilterCondition<'_> { FilterCondition::VectorExists { fid: _, embedder, filter: inner } => { write!(f, "_vectors")?; if let Some(embedder) = embedder { - write!(f, ".{embedder:?}")?; + write!(f, ".{:?}", embedder.value())?; } match inner { - VectorFilter::Fragment(fragment) => write!(f, ".fragments.{fragment:?}"), - VectorFilter::DocumentTemplate => write!(f, ".documentTemplate"), - VectorFilter::UserProvided => write!(f, ".userProvided"), - VectorFilter::Regenerate => write!(f, ".regenerate"), - VectorFilter::None => Ok(()), + VectorFilter::Fragment(fragment) => { + write!(f, ".fragments.{:?}", fragment.value())? + } + VectorFilter::DocumentTemplate => write!(f, ".documentTemplate")?, + VectorFilter::UserProvided => write!(f, ".userProvided")?, + VectorFilter::Regenerate => write!(f, ".regenerate")?, + VectorFilter::None => (), } + write!(f, " EXISTS") } FilterCondition::GeoLowerThan { point, radius } => { write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) @@ -677,6 +680,9 @@ pub mod tests { insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#); // but it also works with other sequences insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}"); + + insta::assert_snapshot!(p(r#"_vectors." valid.name ".fragments."also.. valid! " EXISTS"#), @r#"_vectors." valid.name ".fragments."also.. valid! " EXISTS"#); + insta::assert_snapshot!(p("_vectors.\"\n\t\r\\\"\" EXISTS"), @r#"_vectors."\n\t\r\"" EXISTS"#); } #[test] @@ -739,6 +745,18 @@ pub mod tests { insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY"); insta::assert_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)"); + // Test _vectors EXISTS + _vectors NOT EXITS + insta::assert_snapshot!(p("_vectors EXISTS"), @"_vectors EXISTS"); + insta::assert_snapshot!(p("_vectors.embedderName EXISTS"), @r#"_vectors."embedderName" EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.documentTemplate EXISTS"), @r#"_vectors."embedderName".documentTemplate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.fragments.fragmentName EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#); + insta::assert_snapshot!(p(" _vectors.embedderName.fragments.fragmentName EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#); + insta::assert_snapshot!(p("NOT _vectors EXISTS"), @"NOT (_vectors EXISTS)"); + insta::assert_snapshot!(p(" NOT _vectors EXISTS"), @"NOT (_vectors EXISTS)"); + insta::assert_snapshot!(p(" _vectors NOT EXISTS"), @"NOT (_vectors EXISTS)"); + // Test EXISTS + NOT EXITS insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS"); insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)"); @@ -993,6 +1011,59 @@ pub mod tests { "### ); + insta::assert_snapshot!(p(r#"_vectors _vectors EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors _vectors EXISTS`. + 1:25 _vectors _vectors EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" + The vector filter's embedder is invalid. + 10:30 _vectors. embedderName EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. + 1:30 _vectors .embedderName EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" + The vector filter has leftover tokens. + 22:30 _vectors.embedderName. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" + The vector filter's embedder is invalid. + 30:30 _vectors."embedderName EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" + The vector filter has leftover tokens. + 23:31 _vectors."embedderNam"e EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors.embedderName.documentTemplate. EXISTS"#), @r" + The vector filter has leftover tokens. + 39:47 _vectors.embedderName.documentTemplate. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments EXISTS"#), @r" + The vector filter is missing a fragment name. + 32:39 _vectors.embedderName.fragments EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. EXISTS"#), @r" + The vector filter's fragment is invalid. + 33:40 _vectors.embedderName.fragments. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName.fragments.test test EXISTS`. + 1:49 _vectors.embedderName.fragments.test test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" + The vector filter's fragment is invalid. + 33:45 _vectors.embedderName.fragments. test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments. test EXISTS`. + 1:46 _vectors.embedderName .fragments. test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments.test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments.test EXISTS`. + 1:45 _vectors.embedderName .fragments.test EXISTS + "); + insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. 5:7 NOT OR EXISTS AND EXISTS NOT EXISTS From a92e36ab836626bfbbcc2bdade5d029f0a458ba2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:28:17 +0200 Subject: [PATCH 32/76] Small improvements --- crates/filter-parser/src/condition.rs | 2 +- crates/filter-parser/src/error.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 1e8deef64..4d156c269 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -133,7 +133,7 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> let (input, _) = char('.')(input)?; // From this point, we are certain this is a vector filter, so our errors must be final. - // We could use nom's `cut`` but it's better to be explicit about the errors + // We could use nom's `cut` but it's better to be explicit about the errors let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index cf2419b01..bbf2c8d17 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -78,6 +78,10 @@ pub enum ErrorKind<'a> { GeoBoundingBox, MisusedGeoRadius, MisusedGeoBoundingBox, + VectorFilterLeftover, + VectorFilterInvalidEmbedder, + VectorFilterMissingFragment, + VectorFilterInvalidFragment, InvalidPrimary, InvalidEscapedNumber, ExpectedEof, @@ -93,11 +97,6 @@ pub enum ErrorKind<'a> { InternalError(error::ErrorKind), DepthLimitReached, External(String), - - VectorFilterLeftover, - VectorFilterInvalidEmbedder, - VectorFilterMissingFragment, - VectorFilterInvalidFragment, } impl<'a> Error<'a> { From dbb670a9eeabed8c4d1a465fb8a0223296988086 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:28:58 +0200 Subject: [PATCH 33/76] Remove old split function --- crates/filter-parser/src/lib.rs | 109 -------------------------------- 1 file changed, 109 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 1e342d8d2..608f73290 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -124,16 +124,6 @@ impl<'a> Token<'a> { Err(Error::new_from_kind(self.span, ErrorKind::NonFiniteFloat)) } } - - /// Split the token by a delimiter and return an iterator of tokens. - /// Each token in the iterator will have its own span that corresponds to a slice of the original token's span. - pub fn split(&self, delimiter: &'a str) -> impl Iterator> + '_ { - let original_addr = self.value().as_ptr() as usize; - self.value().split(delimiter).map(move |part| { - let offset = part.as_ptr() as usize - original_addr; - Token::new(self.span.slice(offset..offset + part.len()), Some(part.to_string())) - }) - } } impl<'a> From> for Token<'a> { @@ -1161,103 +1151,4 @@ pub mod tests { let token: Token = s.into(); assert_eq!(token.value(), s); } - - #[test] - fn split() { - let s = "test string that should not be parsed\n newline"; - let token: Token = s.into(); - let parts: Vec<_> = token.split(" ").collect(); - insta::assert_snapshot!(format!("{parts:#?}"), @r#" - [ - Token { - span: LocatedSpan { - offset: 0, - line: 1, - fragment: "test", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "test", - ), - }, - Token { - span: LocatedSpan { - offset: 5, - line: 1, - fragment: "string", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "string", - ), - }, - Token { - span: LocatedSpan { - offset: 12, - line: 1, - fragment: "that", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "that", - ), - }, - Token { - span: LocatedSpan { - offset: 17, - line: 1, - fragment: "should", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "should", - ), - }, - Token { - span: LocatedSpan { - offset: 24, - line: 1, - fragment: "not", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "not", - ), - }, - Token { - span: LocatedSpan { - offset: 28, - line: 1, - fragment: "be", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "be", - ), - }, - Token { - span: LocatedSpan { - offset: 31, - line: 1, - fragment: "parsed\n", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "parsed\n", - ), - }, - Token { - span: LocatedSpan { - offset: 39, - line: 2, - fragment: "newline", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "newline", - ), - }, - ] - "#); - } } From 4264abda23d9b1723a0a0e89d94076bbc10a1214 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:30:36 +0200 Subject: [PATCH 34/76] Remove debugs --- crates/milli/src/search/facet/filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 4e67814d3..1ddfe96c7 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -230,7 +230,7 @@ impl<'a> Filter<'a> { } pub fn use_vector_filter(&self) -> Option<&Token> { - dbg!(self.condition.use_vector_filter()) + self.condition.use_vector_filter() } } @@ -241,7 +241,7 @@ impl<'a> Filter<'a> { let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - let attribute = dbg!(fid.value()); + let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) || attribute == RESERVED_VECTORS_FIELD_NAME From 13d38d59bfacfe7dbefe3de003de3bddee44df4c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:44:11 +0200 Subject: [PATCH 35/76] Remove useless import --- crates/filter-parser/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 608f73290..ae11ccf55 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -60,7 +60,7 @@ use nom::combinator::{cut, eof, map, opt}; use nom::multi::{many0, separated_list1}; use nom::number::complete::recognize_float; use nom::sequence::{delimited, preceded, terminated, tuple}; -use nom::{Finish, Slice}; +use nom::Finish; use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; From 26da478b5b232fee93ccc02837aa1235d561c864 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 17:27:49 +0200 Subject: [PATCH 36/76] Add query vector to response --- .../src/routes/indexes/search_analytics.rs | 1 + .../src/search/federated/perform.rs | 2 + crates/meilisearch/src/search/mod.rs | 9 + crates/meilisearch/tests/search/hybrid.rs | 201 +++++++++++++++++- crates/milli/src/search/hybrid.rs | 12 +- crates/milli/src/search/mod.rs | 2 + crates/milli/src/search/similar.rs | 1 + crates/milli/src/test_index.rs | 1 + 8 files changed, 223 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/search_analytics.rs b/crates/meilisearch/src/routes/indexes/search_analytics.rs index 07f79eba7..2a8e78059 100644 --- a/crates/meilisearch/src/routes/indexes/search_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/search_analytics.rs @@ -224,6 +224,7 @@ impl SearchAggregator { let SearchResult { hits: _, query: _, + query_vector: _, processing_time_ms, hits_info: _, semantic_hit_count: _, diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 5ad64d63c..b747d4da4 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -13,6 +13,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::features::{Network, Remote}; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; +use meilisearch_types::milli::vector::Embedding; use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget, DEFAULT_VALUES_PER_FACET}; use roaring::RoaringBitmap; use tokio::task::JoinHandle; @@ -838,6 +839,7 @@ impl SearchByIndex { document_scores, degraded: query_degraded, used_negative_operator: query_used_negative_operator, + query_vector, } = result; candidates |= query_candidates; diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1c987a70c..1329a6f72 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -841,6 +841,8 @@ pub struct SearchHit { pub struct SearchResult { pub hits: Vec, pub query: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub query_vector: Option>, pub processing_time_ms: u128, #[serde(flatten)] pub hits_info: HitsInfo, @@ -865,6 +867,7 @@ impl fmt::Debug for SearchResult { let SearchResult { hits, query, + query_vector, processing_time_ms, hits_info, facet_distribution, @@ -879,6 +882,9 @@ impl fmt::Debug for SearchResult { debug.field("processing_time_ms", &processing_time_ms); debug.field("hits", &format!("[{} hits returned]", hits.len())); debug.field("query", &query); + if query_vector.is_some() { + debug.field("query_vector", &"[...]"); + } debug.field("hits_info", &hits_info); if *used_negative_operator { debug.field("used_negative_operator", used_negative_operator); @@ -1131,6 +1137,7 @@ pub fn perform_search( document_scores, degraded, used_negative_operator, + query_vector, }, semantic_hit_count, ) = search_from_kind(index_uid, search_kind, search)?; @@ -1221,6 +1228,7 @@ pub fn perform_search( hits: documents, hits_info, query: q.unwrap_or_default(), + query_vector, processing_time_ms: before_search.elapsed().as_millis(), facet_distribution, facet_stats, @@ -1730,6 +1738,7 @@ pub fn perform_similar( document_scores, degraded: _, used_negative_operator: _, + query_vector: _, } = similar.execute().map_err(|err| match err { milli::Error::UserError(milli::UserError::InvalidFilter(_)) => { ResponseError::from_msg(err.to_string(), Code::InvalidSimilarFilter) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index d95e6fb64..b2970f233 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -148,7 +148,70 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}}}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + } + }, + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + } + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + } + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 0 + } + "#); snapshot!(response["semanticHitCount"], @"0"); let (response, code) = index @@ -157,7 +220,73 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.990290343761444 + }, + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9848484848484848 + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9472135901451112 + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 2 + } + "#); snapshot!(response["semanticHitCount"], @"2"); let (response, code) = index @@ -166,7 +295,73 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.990290343761444 + }, + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.974341630935669 + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9472135901451112 + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 3 + } + "#); snapshot!(response["semanticHitCount"], @"3"); } diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index c906e1eb7..e5b4e6787 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -7,7 +7,7 @@ use roaring::RoaringBitmap; use crate::score_details::{ScoreDetails, ScoreValue, ScoringStrategy}; use crate::search::new::{distinct_fid, distinct_single_docid}; use crate::search::SemanticSearch; -use crate::vector::SearchQuery; +use crate::vector::{Embedding, SearchQuery}; use crate::{Index, MatchingWords, Result, Search, SearchResult}; struct ScoreWithRatioResult { @@ -16,6 +16,7 @@ struct ScoreWithRatioResult { document_scores: Vec<(u32, ScoreWithRatio)>, degraded: bool, used_negative_operator: bool, + query_vector: Option, } type ScoreWithRatio = (Vec, f32); @@ -85,6 +86,7 @@ impl ScoreWithRatioResult { document_scores, degraded: results.degraded, used_negative_operator: results.used_negative_operator, + query_vector: results.query_vector, } } @@ -186,6 +188,7 @@ impl ScoreWithRatioResult { degraded: vector_results.degraded | keyword_results.degraded, used_negative_operator: vector_results.used_negative_operator | keyword_results.used_negative_operator, + query_vector: vector_results.query_vector, }, semantic_hit_count, )) @@ -264,7 +267,7 @@ impl Search<'_> { }; search.semantic = Some(SemanticSearch { - vector: Some(vector_query), + vector: Some(vector_query.clone()), embedder_name, embedder, quantized, @@ -277,7 +280,7 @@ impl Search<'_> { let keyword_results = ScoreWithRatioResult::new(keyword_results, 1.0 - semantic_ratio); let vector_results = ScoreWithRatioResult::new(vector_results, semantic_ratio); - let (merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( + let (mut merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( vector_results, keyword_results, self.offset, @@ -286,6 +289,7 @@ impl Search<'_> { search.index, search.rtxn, )?; + merge_results.query_vector = Some(vector_query); assert!(merge_results.documents_ids.len() <= self.limit); Ok((merge_results, Some(semantic_hit_count))) } @@ -321,6 +325,7 @@ fn return_keyword_results( mut document_scores, degraded, used_negative_operator, + query_vector, }: SearchResult, ) -> (SearchResult, Option) { let (documents_ids, document_scores) = if offset >= documents_ids.len() || @@ -347,6 +352,7 @@ fn return_keyword_results( document_scores, degraded, used_negative_operator, + query_vector, }, Some(0), ) diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 97d542524..cd0d5bc9b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -295,6 +295,7 @@ impl<'a> Search<'a> { documents_ids, degraded, used_negative_operator, + query_vector: None, }) } } @@ -353,6 +354,7 @@ pub struct SearchResult { pub document_scores: Vec>, pub degraded: bool, pub used_negative_operator: bool, + pub query_vector: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 903b5fcf9..2235f6436 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -130,6 +130,7 @@ impl<'a> Similar<'a> { document_scores, degraded: false, used_negative_operator: false, + query_vector: None, }) } } diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6bb6b1345..d174319d0 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -1097,6 +1097,7 @@ fn bug_3021_fourth() { mut documents_ids, degraded: _, used_negative_operator: _, + query_vector: _, } = search.execute().unwrap(); let primary_key_id = index.fields_ids_map(&rtxn).unwrap().id("primary_key").unwrap(); documents_ids.sort_unstable(); From a7fe2abca4f6425967c55c6789940ad1dd303f04 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 11:45:51 +0200 Subject: [PATCH 37/76] Implement for multi-search --- .../src/search/federated/perform.rs | 37 +++- .../meilisearch/src/search/federated/types.rs | 9 + crates/meilisearch/src/search/mod.rs | 2 +- .../meilisearch/tests/search/multi/proxy.rs | 208 +++++++++++++++++- crates/milli/src/search/hybrid.rs | 18 +- crates/milli/src/search/mod.rs | 70 ++++-- 6 files changed, 318 insertions(+), 26 deletions(-) diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index b747d4da4..4c66e5f68 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -47,6 +47,7 @@ pub async fn perform_federated_search( let deadline = before_search + std::time::Duration::from_secs(9); let required_hit_count = federation.limit + federation.offset; + let retrieve_vectors = queries.iter().any(|q| q.retrieve_vectors); let network = index_scheduler.network(); @@ -92,6 +93,7 @@ pub async fn perform_federated_search( federation, mut semantic_hit_count, mut results_by_index, + mut query_vectors, previous_query_data: _, facet_order, } = search_by_index; @@ -123,7 +125,26 @@ pub async fn perform_federated_search( .map(|hit| hit.hit()) .collect(); - // 3.3. merge facets + // 3.3. merge query vectors + let query_vectors = if retrieve_vectors { + for remote_results in remote_results.iter_mut() { + if let Some(remote_vectors) = remote_results.query_vectors.take() { + for (key, value) in remote_vectors.into_iter() { + debug_assert!( + !query_vectors.contains_key(&key), + "Query vector for query {key} already exists" + ); + query_vectors.insert(key, value); + } + } + } + + Some(query_vectors) + } else { + None + }; + + // 3.4. merge facets let (facet_distribution, facet_stats, facets_by_index) = facet_order.merge(federation.merge_facets, remote_results, facets); @@ -141,6 +162,7 @@ pub async fn perform_federated_search( offset: federation.offset, estimated_total_hits, }, + query_vectors, semantic_hit_count, degraded, used_negative_operator, @@ -409,6 +431,7 @@ fn merge_metadata( hits: _, processing_time_ms, hits_info, + query_vectors: _, semantic_hit_count: _, facet_distribution: _, facet_stats: _, @@ -658,6 +681,7 @@ struct SearchByIndex { // Then when merging, we'll update its value if there is any semantic hit semantic_hit_count: Option, results_by_index: Vec, + query_vectors: BTreeMap, previous_query_data: Option<(RankingRules, usize, String)>, // remember the order and name of first index for each facet when merging with index settings // to detect if the order is inconsistent for a facet. @@ -675,6 +699,7 @@ impl SearchByIndex { federation, semantic_hit_count: None, results_by_index: Vec::with_capacity(index_count), + query_vectors: BTreeMap::new(), previous_query_data: None, } } @@ -842,6 +867,16 @@ impl SearchByIndex { query_vector, } = result; + if query.retrieve_vectors { + if let Some(query_vector) = query_vector { + debug_assert!( + !self.query_vectors.contains_key(&query_index), + "Query vector for query {query_index} already exists" + ); + self.query_vectors.insert(query_index, query_vector); + } + } + candidates |= query_candidates; degraded |= query_degraded; used_negative_operator |= query_used_negative_operator; diff --git a/crates/meilisearch/src/search/federated/types.rs b/crates/meilisearch/src/search/federated/types.rs index 3cf28c815..9c96fe768 100644 --- a/crates/meilisearch/src/search/federated/types.rs +++ b/crates/meilisearch/src/search/federated/types.rs @@ -18,6 +18,7 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use super::super::{ComputedFacets, FacetStats, HitsInfo, SearchHit, SearchQueryWithIndex}; +use crate::milli::vector::Embedding; pub const DEFAULT_FEDERATED_WEIGHT: f64 = 1.0; @@ -117,6 +118,9 @@ pub struct FederatedSearchResult { #[serde(flatten)] pub hits_info: HitsInfo, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_vectors: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub semantic_hit_count: Option, @@ -144,6 +148,7 @@ impl fmt::Debug for FederatedSearchResult { hits, processing_time_ms, hits_info, + query_vectors, semantic_hit_count, degraded, used_negative_operator, @@ -158,6 +163,10 @@ impl fmt::Debug for FederatedSearchResult { debug.field("processing_time_ms", &processing_time_ms); debug.field("hits", &format!("[{} hits returned]", hits.len())); debug.field("hits_info", &hits_info); + if let Some(query_vectors) = query_vectors { + let known = query_vectors.len(); + debug.field("query_vectors", &format!("[{known} known vectors]")); + } if *used_negative_operator { debug.field("used_negative_operator", used_negative_operator); } diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1329a6f72..e08a13a7b 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1020,7 +1020,7 @@ pub fn prepare_search<'t>( .map_err(milli::Error::from)? } }; - search.semantic( + search.semantic_auto_embedded( embedder_name.clone(), embedder.clone(), *quantized, diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 311f69d9e..8f2741202 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -2,8 +2,9 @@ use std::sync::Arc; use actix_http::StatusCode; use meili_snap::{json_string, snapshot}; -use wiremock::matchers::AnyMatcher; -use wiremock::{Mock, MockServer, ResponseTemplate}; +use wiremock::matchers::method; +use wiremock::matchers::{path, AnyMatcher}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::{Server, Value, SCORE_DOCUMENTS}; use crate::json; @@ -415,6 +416,209 @@ async fn remote_sharding() { "###); } +#[actix_rt::test] +async fn remote_sharding_retrieve_vectors() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + let ms2 = Server::new().await; + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let index2 = ms2.index("test"); + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms2.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms2", + "remotes": {} + } + "###); + + // setup embedders + + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/")) + .respond_with(move |req: &Request| { + println!("Received request: {:?}", req); + let text = req.body_json::().unwrap().to_lowercase(); + let patterns = [ + ("batman", [1.0, 0.0, 0.0]), + ("dark", [0.0, 0.1, 0.0]), + ("knight", [0.1, 0.1, 0.0]), + ("returns", [0.0, 0.0, 0.2]), + ("part", [0.05, 0.1, 0.0]), + ("1", [0.3, 0.05, 0.0]), + ("2", [0.2, 0.05, 0.0]), + ]; + let mut embedding = vec![0.; 3]; + for (pattern, vector) in patterns { + if text.contains(pattern) { + for (i, v) in vector.iter().enumerate() { + embedding[i] += v; + } + } + } + ResponseTemplate::new(200).set_body_json(json!({ "data": embedding })) + }) + .mount(&mock_server) + .await; + let url = mock_server.uri(); + + for (server, index) in [(&ms0, &index0), (&ms1, &index1), (&ms2, &index2)] { + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{text}}", + "response": { "data": "{{embedding}}" }, + "documentTemplate": "{{doc.name}}", + }, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + } + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + let ms2 = Arc::new(ms2); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + let rms2 = LocalMeili::new(ms2.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + "ms2": { + "url": rms2.url() + } + }}); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms2.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 0.0, + 0.0, + 0.2 + ], + "1": [ + 0.0, + 0.0, + 0.2 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); +} + #[actix_rt::test] async fn error_unregistered_remote() { let ms0 = Server::new().await; diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index e5b4e6787..547d66409 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -230,7 +230,14 @@ impl Search<'_> { } // no embedder, no semantic search - let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic + let Some(SemanticSearch { + vector, + mut auto_embedded, + embedder_name, + embedder, + quantized, + media, + }) = semantic else { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); }; @@ -253,7 +260,10 @@ impl Search<'_> { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); match embedder.embed_search(query, Some(deadline)) { - Ok(embedding) => embedding, + Ok(embedding) => { + auto_embedded = true; + embedding + } Err(error) => { tracing::error!(error=%error, "Embedding failed"); return Ok(return_keyword_results( @@ -268,6 +278,7 @@ impl Search<'_> { search.semantic = Some(SemanticSearch { vector: Some(vector_query.clone()), + auto_embedded, embedder_name, embedder, quantized, @@ -280,7 +291,7 @@ impl Search<'_> { let keyword_results = ScoreWithRatioResult::new(keyword_results, 1.0 - semantic_ratio); let vector_results = ScoreWithRatioResult::new(vector_results, semantic_ratio); - let (mut merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( + let (merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( vector_results, keyword_results, self.offset, @@ -289,7 +300,6 @@ impl Search<'_> { search.index, search.rtxn, )?; - merge_results.query_vector = Some(vector_query); assert!(merge_results.documents_ids.len() <= self.limit); Ok((merge_results, Some(semantic_hit_count))) } diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index cd0d5bc9b..de0514b0b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -32,6 +32,7 @@ pub mod similar; #[derive(Debug, Clone)] pub struct SemanticSearch { vector: Option>, + auto_embedded: bool, media: Option, embedder_name: String, embedder: Arc, @@ -97,7 +98,33 @@ impl<'a> Search<'a> { vector: Option, media: Option, ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media }); + self.semantic = Some(SemanticSearch { + embedder_name, + auto_embedded: false, + embedder, + quantized, + vector, + media, + }); + self + } + + pub fn semantic_auto_embedded( + &mut self, + embedder_name: String, + embedder: Arc, + quantized: bool, + vector: Option, + media: Option, + ) -> &mut Search<'a> { + self.semantic = Some(SemanticSearch { + embedder_name, + auto_embedded: true, + embedder, + quantized, + vector, + media, + }); self } @@ -225,6 +252,7 @@ impl<'a> Search<'a> { } let universe = filtered_universe(ctx.index, ctx.txn, &self.filter)?; + let mut query_vector = None; let PartialSearchResult { located_query_terms, candidates, @@ -235,26 +263,32 @@ impl<'a> Search<'a> { } = match self.semantic.as_ref() { Some(SemanticSearch { vector: Some(vector), + auto_embedded, embedder_name, embedder, quantized, media: _, - }) => execute_vector_search( - &mut ctx, - vector, - self.scoring_strategy, - universe, - &self.sort_criteria, - &self.distinct, - self.geo_param, - self.offset, - self.limit, - embedder_name, - embedder, - *quantized, - self.time_budget.clone(), - self.ranking_score_threshold, - )?, + }) => { + if *auto_embedded { + query_vector = Some(vector.clone()); + } + execute_vector_search( + &mut ctx, + vector, + self.scoring_strategy, + universe, + &self.sort_criteria, + &self.distinct, + self.geo_param, + self.offset, + self.limit, + embedder_name, + embedder, + *quantized, + self.time_budget.clone(), + self.ranking_score_threshold, + )? + } _ => execute_search( &mut ctx, self.query.as_deref(), @@ -295,7 +329,7 @@ impl<'a> Search<'a> { documents_ids, degraded, used_negative_operator, - query_vector: None, + query_vector, }) } } From d243504296eba4e3eb0003c325b4162c8ba44502 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 11:58:34 +0200 Subject: [PATCH 38/76] Improve test --- .../meilisearch/tests/search/multi/proxy.rs | 314 +++++++++++++++++- 1 file changed, 304 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 8f2741202..920d88234 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -543,13 +543,13 @@ async fn remote_sharding_retrieve_vectors() { let (_response, status_code) = ms2.set_network(network.clone()).await; snapshot!(status_code, @"200 OK"); - // perform multi-search - let query = "badman returns"; + // multi vector search: one query per remote + let request = json!({ "federation": {}, "queries": [ { - "q": query, + "q": "batman", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -561,7 +561,7 @@ async fn remote_sharding_retrieve_vectors() { } }, { - "q": query, + "q": "dark knight", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -573,7 +573,7 @@ async fn remote_sharding_retrieve_vectors() { } }, { - "q": query, + "q": "returns", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -598,14 +598,14 @@ async fn remote_sharding_retrieve_vectors() { "estimatedTotalHits": 0, "queryVectors": { "0": [ + 1.0, 0.0, - 0.0, - 0.2 + 0.0 ], "1": [ - 0.0, - 0.0, - 0.2 + 0.1, + 0.2, + 0.0 ], "2": [ 0.0, @@ -617,6 +617,300 @@ async fn remote_sharding_retrieve_vectors() { "remoteErrors": {} } "#); + + // multi vector search: two local queries, one remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi vector search: two queries on the same remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi search: two vector, one keyword + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 0.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi vector search: no local queries, all remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "remoteErrors": {} + } + "#); } #[actix_rt::test] From a439f57d7076682e449f1102cf275429a6a54b62 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 13:41:31 +0200 Subject: [PATCH 39/76] Update tests --- crates/meilisearch/tests/search/hybrid.rs | 4 ---- crates/meilisearch/tests/search/multi/mod.rs | 15 +++++++++++++-- .../tests/vector/binary_quantized.rs | 9 +++++++-- crates/meilisearch/tests/vector/mod.rs | 18 ++++++++++++++---- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index b2970f233..fce94dd7f 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -201,10 +201,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/search/multi/mod.rs b/crates/meilisearch/tests/search/multi/mod.rs index b9eed56da..3b0d9380e 100644 --- a/crates/meilisearch/tests/search/multi/mod.rs +++ b/crates/meilisearch/tests/search/multi/mod.rs @@ -3703,7 +3703,7 @@ async fn federation_vector_two_indexes() { ]})) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r###" + snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r#" { "hits": [ { @@ -3911,9 +3911,20 @@ async fn federation_vector_two_indexes() { "limit": 20, "offset": 0, "estimatedTotalHits": 8, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.5 + ], + "1": [ + 0.8, + 0.6 + ] + }, "semanticHitCount": 6 } - "###); + "#); // hybrid search, distinct embedder let (response, code) = server diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index 6fcfa3563..e0fa9a37c 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -323,15 +323,20 @@ async fn binary_quantize_clear_documents() { // Make sure the arroy DB has been cleared let (documents, _code) = index.search_post(json!({ "hybrid": { "embedder": "manual" }, "vector": [1, 1, 1] })).await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 0, "semanticHitCount": 0 } - "###); + "#); } diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index ca2ecc998..5d056e365 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -685,17 +685,22 @@ async fn clear_documents() { // Make sure the arroy DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "manual"} })).await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 0, "semanticHitCount": 0 } - "###); + "#); } #[actix_rt::test] @@ -739,7 +744,7 @@ async fn add_remove_one_vector_4588() { json!({"vector": [1, 1, 1], "hybrid": {"semanticRatio": 1.0, "embedder": "manual"} }), ) .await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [ { @@ -748,13 +753,18 @@ async fn add_remove_one_vector_4588() { } ], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 1, "semanticHitCount": 1 } - "###); + "#); let (documents, _code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) From 10567b150cc95aed2cc102c85b71f03aca03785c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 14:25:35 +0200 Subject: [PATCH 40/76] Continue updating tests --- crates/meilisearch/tests/search/hybrid.rs | 8 -------- crates/meilisearch/tests/search/multi/mod.rs | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index fce94dd7f..172242e47 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -272,10 +272,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -347,10 +343,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/search/multi/mod.rs b/crates/meilisearch/tests/search/multi/mod.rs index 3b0d9380e..16ee3906e 100644 --- a/crates/meilisearch/tests/search/multi/mod.rs +++ b/crates/meilisearch/tests/search/multi/mod.rs @@ -3934,7 +3934,7 @@ async fn federation_vector_two_indexes() { ]})) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r###" + snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r#" { "hits": [ { @@ -4150,9 +4150,20 @@ async fn federation_vector_two_indexes() { "limit": 20, "offset": 0, "estimatedTotalHits": 8, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.5 + ], + "1": [ + -1.0, + 0.6 + ] + }, "semanticHitCount": 8 } - "###); + "#); } #[actix_rt::test] From 6c3dd83ae528a8b82614880c0206cd7886a187c7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 09:03:48 +0200 Subject: [PATCH 41/76] Fix old test --- crates/milli/src/test_index.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 0ec348301..a6ed9ad91 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -1340,11 +1340,10 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { assert!(results.candidates.is_empty()); let mut search = index.search(&rtxn); - let results = search - .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) - .execute() - .unwrap_err(); - assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); + let results = + dbg!(search.filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()).execute()) + .unwrap(); + assert!(results.candidates.is_empty()); index .update_settings(|settings| { @@ -1375,6 +1374,6 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap_err(); - assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); + .unwrap(); + assert!(results.candidates.is_empty()); } From 66b6e47494c0d6809c0c78fd7f794dcd673afc62 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 10:52:21 +0200 Subject: [PATCH 42/76] Remove warning --- crates/milli/src/test_index.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index a6ed9ad91..12ac4e158 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -19,9 +19,7 @@ use crate::update::{ }; use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; use crate::vector::RuntimeEmbedders; -use crate::{ - db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, UserError, -}; +use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; pub(crate) struct TempIndex { pub inner: Index, From 3580b3a4ef5370c5168954378e2aa4df9464b6af Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 10:56:54 +0200 Subject: [PATCH 43/76] Remove userProvided from fragments --- crates/meilisearch/tests/search/filters.rs | 5 +---- crates/milli/src/search/facet/filter_vector.rs | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 67f9ebb71..209261f70 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -930,9 +930,6 @@ async fn vector_filter_specific_fragment() { { "name": "kefir" }, - { - "name": "echo" - }, { "name": "intel" }, @@ -944,7 +941,7 @@ async fn vector_filter_specific_fragment() { "processingTimeMs": "[duration]", "limit": 20, "offset": 0, - "estimatedTotalHits": 4 + "estimatedTotalHits": 3 } "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 2ddd801ed..2cde3aaa7 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -119,7 +119,10 @@ fn evaluate_inner( .collect(), })?; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| { + bitmap.clone() - user_provided_docsids + })? } VectorFilter::DocumentTemplate => { if !embedding_config.fragments.as_slice().is_empty() { From 223df5a43346ea4bdd2c63218c9e0985d21a9f30 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:02:59 +0200 Subject: [PATCH 44/76] Remove incorrect break --- crates/milli/src/vector/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f64223e41..088d98b72 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -556,9 +556,6 @@ impl ArroyWrapper { for reader in self.readers(rtxn, self.quantized_db()) { let reader = reader?; let documents = reader.item_ids(); - if documents.is_empty() { - break; - } stats.documents |= documents; stats.number_of_embeddings += documents.len(); } @@ -566,9 +563,6 @@ impl ArroyWrapper { for reader in self.readers(rtxn, self.angular_db()) { let reader = reader?; let documents = reader.item_ids(); - if documents.is_empty() { - break; - } stats.documents |= documents; stats.number_of_embeddings += documents.len(); } From 93864009cc68a5af622ef0ce2f6d7f96ec85f332 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:04:08 +0200 Subject: [PATCH 45/76] Rename variable with typo Co-Authored-By: Louis Dureuil --- crates/milli/src/search/facet/filter_vector.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 2cde3aaa7..625bd5dde 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -119,9 +119,9 @@ fn evaluate_inner( .collect(), })?; - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| { - bitmap.clone() - user_provided_docsids + bitmap.clone() - user_provided_docids })? } VectorFilter::DocumentTemplate => { @@ -129,14 +129,14 @@ fn evaluate_inner( return Ok(RoaringBitmap::new()); } - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() + stats.documents - user_provided_docids.clone() } VectorFilter::UserProvided => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docids.clone() } VectorFilter::Regenerate => { let mut stats = ArroyStats::default(); From 60acdf8574e9e0d2e7e26eb553dd5410554f8b9c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:05:16 +0200 Subject: [PATCH 46/76] Fix grammar Co-Authored-By: Louis Dureuil --- crates/meilisearch/tests/search/filters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 209261f70..76f252a6d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -788,7 +788,7 @@ async fn vector_filter_missing_fragment() { } #[actix_rt::test] -async fn vector_filter_non_existant_embedder() { +async fn vector_filter_nonexistent_embedder() { let index = shared_index_for_fragments().await; let (value, _code) = index From 2121819c66a0d66b9627a46962316045f77a1af0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 14:18:45 +0200 Subject: [PATCH 47/76] Fix tests --- crates/meilisearch/tests/search/filters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 76f252a6d..8e3ee9249 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1035,7 +1035,7 @@ async fn vector_filter_document_template() { ]); let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); - index.wait_task(value.uid()).await.succeeded(); + server.wait_task(value.uid()).await.succeeded(); let (value, _code) = index .search_post(json!({ From c385cf985b679207781a17e65ef20df2bb1a39b6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 15:55:31 +0200 Subject: [PATCH 48/76] Fix tests --- crates/meilisearch/src/search/mod.rs | 1 + crates/meilisearch/tests/vector/binary_quantized.rs | 5 ----- crates/meilisearch/tests/vector/mod.rs | 10 ---------- crates/milli/src/search/hybrid.rs | 1 + crates/milli/src/search/mod.rs | 11 ++++++++++- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index d4775c66a..c681a11a5 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1056,6 +1056,7 @@ pub fn prepare_search<'t>( .map(|x| x as usize) .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS); + search.retrieve_vectors(query.retrieve_vectors); search.exhaustive_number_hits(is_finite_pagination); search.max_total_hits(Some(max_total_hits)); search.scoring_strategy( diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index e0fa9a37c..adb0da441 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -327,11 +327,6 @@ async fn binary_quantize_clear_documents() { { "hits": [], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 0eb1f063e..551b82178 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -690,11 +690,6 @@ async fn clear_documents() { { "hits": [], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -754,11 +749,6 @@ async fn add_remove_one_vector_4588() { } ], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index 353c69dab..75ac547b4 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -212,6 +212,7 @@ impl Search<'_> { terms_matching_strategy: self.terms_matching_strategy, scoring_strategy: ScoringStrategy::Detailed, words_limit: self.words_limit, + retrieve_vectors: self.retrieve_vectors, exhaustive_number_hits: self.exhaustive_number_hits, max_total_hits: self.max_total_hits, rtxn: self.rtxn, diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 2192e842f..e7871031b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -53,6 +53,7 @@ pub struct Search<'a> { terms_matching_strategy: TermsMatchingStrategy, scoring_strategy: ScoringStrategy, words_limit: usize, + retrieve_vectors: bool, exhaustive_number_hits: bool, max_total_hits: Option, rtxn: &'a heed::RoTxn<'a>, @@ -76,6 +77,7 @@ impl<'a> Search<'a> { geo_param: GeoSortParameter::default(), terms_matching_strategy: TermsMatchingStrategy::default(), scoring_strategy: Default::default(), + retrieve_vectors: false, exhaustive_number_hits: false, max_total_hits: None, words_limit: 10, @@ -188,6 +190,11 @@ impl<'a> Search<'a> { self } + pub fn retrieve_vectors(&mut self, retrieve_vectors: bool) -> &mut Search<'a> { + self.retrieve_vectors = retrieve_vectors; + self + } + /// Forces the search to exhaustively compute the number of candidates, /// this will increase the search time but allows finite pagination. pub fn exhaustive_number_hits(&mut self, exhaustive_number_hits: bool) -> &mut Search<'a> { @@ -277,7 +284,7 @@ impl<'a> Search<'a> { quantized, media: _, }) => { - if *auto_embedded { + if *auto_embedded && self.retrieve_vectors { query_vector = Some(vector.clone()); } execute_vector_search( @@ -359,6 +366,7 @@ impl fmt::Debug for Search<'_> { terms_matching_strategy, scoring_strategy, words_limit, + retrieve_vectors, exhaustive_number_hits, max_total_hits, rtxn: _, @@ -379,6 +387,7 @@ impl fmt::Debug for Search<'_> { .field("searchable_attributes", searchable_attributes) .field("terms_matching_strategy", terms_matching_strategy) .field("scoring_strategy", scoring_strategy) + .field("retrieve_vectors", retrieve_vectors) .field("exhaustive_number_hits", exhaustive_number_hits) .field("max_total_hits", max_total_hits) .field("words_limit", words_limit) From 74992560b00bb34a0f369167b0f9447696072f19 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 7 Aug 2025 09:28:45 +0200 Subject: [PATCH 49/76] Simplify conditions --- crates/meilisearch/src/search/mod.rs | 2 +- crates/meilisearch/tests/search/hybrid.rs | 12 ++++++++++++ crates/milli/src/search/hybrid.rs | 7 +------ crates/milli/src/search/mod.rs | 24 +---------------------- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index c681a11a5..bb406aed9 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1020,7 +1020,7 @@ pub fn prepare_search<'t>( .map_err(milli::Error::from)? } }; - search.semantic_auto_embedded( + search.semantic( embedder_name.clone(), embedder.clone(), *quantized, diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index 172242e47..b2970f233 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -201,6 +201,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -272,6 +276,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -343,6 +351,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index 75ac547b4..c4d440043 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -234,7 +234,6 @@ impl Search<'_> { // no embedder, no semantic search let Some(SemanticSearch { vector, - mut auto_embedded, embedder_name, embedder, quantized, @@ -262,10 +261,7 @@ impl Search<'_> { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); match embedder.embed_search(query, Some(deadline)) { - Ok(embedding) => { - auto_embedded = true; - embedding - } + Ok(embedding) => embedding, Err(error) => { tracing::error!(error=%error, "Embedding failed"); return Ok(return_keyword_results( @@ -280,7 +276,6 @@ impl Search<'_> { search.semantic = Some(SemanticSearch { vector: Some(vector_query.clone()), - auto_embedded, embedder_name, embedder, quantized, diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index e7871031b..155a0ecf7 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -33,7 +33,6 @@ pub mod similar; #[derive(Debug, Clone)] pub struct SemanticSearch { vector: Option>, - auto_embedded: bool, media: Option, embedder_name: String, embedder: Arc, @@ -105,26 +104,6 @@ impl<'a> Search<'a> { ) -> &mut Search<'a> { self.semantic = Some(SemanticSearch { embedder_name, - auto_embedded: false, - embedder, - quantized, - vector, - media, - }); - self - } - - pub fn semantic_auto_embedded( - &mut self, - embedder_name: String, - embedder: Arc, - quantized: bool, - vector: Option, - media: Option, - ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { - embedder_name, - auto_embedded: true, embedder, quantized, vector, @@ -278,13 +257,12 @@ impl<'a> Search<'a> { } = match self.semantic.as_ref() { Some(SemanticSearch { vector: Some(vector), - auto_embedded, embedder_name, embedder, quantized, media: _, }) => { - if *auto_embedded && self.retrieve_vectors { + if self.retrieve_vectors { query_vector = Some(vector.clone()); } execute_vector_search( From 5df125cbb78fdaf82b5941ad1f6bb2d968382c09 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 7 Aug 2025 09:31:05 +0200 Subject: [PATCH 50/76] Format --- crates/milli/src/search/hybrid.rs | 8 +------- crates/milli/src/search/mod.rs | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index c4d440043..1535c73ba 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -232,13 +232,7 @@ impl Search<'_> { } // no embedder, no semantic search - let Some(SemanticSearch { - vector, - embedder_name, - embedder, - quantized, - media, - }) = semantic + let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic else { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); }; diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 155a0ecf7..2ae931ff5 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -102,13 +102,7 @@ impl<'a> Search<'a> { vector: Option, media: Option, ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { - embedder_name, - embedder, - quantized, - vector, - media, - }); + self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media }); self } From 0881810780c0398645f1d09b7b2b603f1068ee05 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:09:54 +0200 Subject: [PATCH 51/76] Add CI to publish OpenAPI file --- ...inaries.yml => publish-release-assets.yml} | 30 ++++++++++++- .gitignore | 3 ++ Cargo.lock | 11 +++++ Cargo.toml | 1 + crates/openapi-generator/Cargo.toml | 12 ++++++ crates/openapi-generator/src/main.rs | 42 +++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) rename .github/workflows/{publish-binaries.yml => publish-release-assets.yml} (87%) create mode 100644 crates/openapi-generator/Cargo.toml create mode 100644 crates/openapi-generator/src/main.rs diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-release-assets.yml similarity index 87% rename from .github/workflows/publish-binaries.yml rename to .github/workflows/publish-release-assets.yml index 27d8c3610..204480887 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-release-assets.yml @@ -1,4 +1,4 @@ -name: Publish binaries to GitHub release +name: Publish assets to GitHub release on: workflow_dispatch: @@ -184,3 +184,31 @@ jobs: file: target/${{ matrix.target }}/release/meilisearch asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} + + publish-openapi: + name: Publish OpenAPI file + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Generate OpenAPI specification + run: | + cd crates/openapi-generator + cargo run --release -- --pretty --output ../../meilisearch.json + + - name: Upload OpenAPI to Release + # No need to upload for dry run (cron) + if: github.event_name == 'release' + uses: svenstaro/upload-release-action@2.11.2 + with: + repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} + file: ./meilisearch.json + asset_name: meilisearch-openapi.json + tag: ${{ github.ref }} diff --git a/.gitignore b/.gitignore index 44cfa8f75..d9a945b88 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ crates/meilisearch/db.snapshot # Fuzzcheck data for the facet indexing fuzz test crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ + +# OpenAPI +meilisearch.json diff --git a/Cargo.lock b/Cargo.lock index 6894e4856..061cefb3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4338,6 +4338,17 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openapi-generator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "meilisearch", + "serde_json", + "utoipa", +] + [[package]] name = "openssl-probe" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index 1fa86c671..dd2747c5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/tracing-trace", "crates/xtask", "crates/build-info", + "crates/openapi-generator", ] [workspace.package] diff --git a/crates/openapi-generator/Cargo.toml b/crates/openapi-generator/Cargo.toml new file mode 100644 index 000000000..87c41625e --- /dev/null +++ b/crates/openapi-generator/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "openapi-generator" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +meilisearch = { path = "../meilisearch" } +serde_json = "1.0" +clap = { version = "4.5.40", features = ["derive"] } +anyhow = "1.0.98" +utoipa = "5.4.0" \ No newline at end of file diff --git a/crates/openapi-generator/src/main.rs b/crates/openapi-generator/src/main.rs new file mode 100644 index 000000000..5630a6fea --- /dev/null +++ b/crates/openapi-generator/src/main.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use clap::Parser; +use meilisearch::routes::MeilisearchApi; +use utoipa::OpenApi; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "openapi-generator")] +#[command(about = "Generate OpenAPI specification for Meilisearch")] +struct Cli { + /// Output file path (default: meilisearch.json) + #[arg(short, long, value_name = "FILE")] + output: Option, + + /// Pretty print the JSON output + #[arg(short, long)] + pretty: bool, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + // Generate the OpenAPI specification + let openapi = MeilisearchApi::openapi(); + + // Determine output path + let output_path = cli.output.unwrap_or_else(|| PathBuf::from("meilisearch.json")); + + // Serialize to JSON + let json = if cli.pretty { + serde_json::to_string_pretty(&openapi)? + } else { + serde_json::to_string(&openapi)? + }; + + // Write to file + std::fs::write(&output_path, json)?; + + println!("OpenAPI specification written to: {}", output_path.display()); + + Ok(()) +} From 3c583ce7a4f23d86aa8748bfa930c9cdffa45368 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:10:39 +0200 Subject: [PATCH 52/76] Fix linting --- crates/openapi-generator/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openapi-generator/Cargo.toml b/crates/openapi-generator/Cargo.toml index 87c41625e..652f6fc57 100644 --- a/crates/openapi-generator/Cargo.toml +++ b/crates/openapi-generator/Cargo.toml @@ -9,4 +9,4 @@ meilisearch = { path = "../meilisearch" } serde_json = "1.0" clap = { version = "4.5.40", features = ["derive"] } anyhow = "1.0.98" -utoipa = "5.4.0" \ No newline at end of file +utoipa = "5.4.0" From 100a6f96e45f737e273860fab8e7a15134576de4 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:11:23 +0200 Subject: [PATCH 53/76] Minor change --- .github/workflows/publish-release-assets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index 204480887..f9db0c91b 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -185,7 +185,7 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-openapi: + publish-openapi-file: name: Publish OpenAPI file runs-on: ubuntu-latest steps: From c5b325de30efa1ce3d9d3e784af362d1c45781d3 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:15:42 +0200 Subject: [PATCH 54/76] Fix rustfmt --- crates/openapi-generator/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/openapi-generator/src/main.rs b/crates/openapi-generator/src/main.rs index 5630a6fea..a6196f771 100644 --- a/crates/openapi-generator/src/main.rs +++ b/crates/openapi-generator/src/main.rs @@ -1,8 +1,9 @@ +use std::path::PathBuf; + use anyhow::Result; use clap::Parser; use meilisearch::routes::MeilisearchApi; use utoipa::OpenApi; -use std::path::PathBuf; #[derive(Parser)] #[command(name = "openapi-generator")] From a69af611e30565f53f35a78f08d6986f4fe11615 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:29:52 +0200 Subject: [PATCH 55/76] Add documentation --- .gitignore | 3 --- CONTRIBUTING.md | 12 +++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d9a945b88..44cfa8f75 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,3 @@ crates/meilisearch/db.snapshot # Fuzzcheck data for the facet indexing fuzz test crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ - -# OpenAPI -meilisearch.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72a91a765..7f718c899 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,12 +107,18 @@ Run `cargo xtask --help` from the root of the repository to find out what is ava To update the openAPI file in the code, see [sprint_issue.md](https://github.com/meilisearch/meilisearch/blob/main/.github/ISSUE_TEMPLATE/sprint_issue.md#reminders-when-modifying-the-api). -If you want to update the openAPI file on the [open-api repository](https://github.com/meilisearch/open-api): -- Pull the latest version of the latest rc of Meilisearch `git checkout release-vX.Y.Z; git pull` +If you want to generate OpenAPI file manually: + +With swagger: - Starts Meilisearch with the `swagger` feature flag: `cargo run --features swagger` - On a browser, open the following URL: http://localhost:7700/scalar - Click the « Download openAPI file » -- Open a PR replacing [this file](https://github.com/meilisearch/open-api/blob/main/open-api.json) with the one downloaded + +With the internal crate: +```bash +cd crates/openapi-generator +cargo run --release -- --pretty --output meilisearch.json +``` ### Logging From 3c84010403607a63e149ebd67d0c2d67c6e5ddb1 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:31:30 +0200 Subject: [PATCH 56/76] Minor change in CI manifest --- .github/workflows/publish-release-assets.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index f9db0c91b..cb70b23fc 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -191,18 +191,15 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup Rust uses: actions-rs/toolchain@v1 with: toolchain: stable override: true - - - name: Generate OpenAPI specification + - name: Generate OpenAPI file run: | cd crates/openapi-generator cargo run --release -- --pretty --output ../../meilisearch.json - - name: Upload OpenAPI to Release # No need to upload for dry run (cron) if: github.event_name == 'release' From 14b1a3300bb3a94fd55e9b5031ddcec2772a172e Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 10:07:34 +0200 Subject: [PATCH 57/76] Fix indentation --- .github/workflows/publish-release-assets.yml | 48 ++++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index cb70b23fc..ec0d36711 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -185,27 +185,27 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-openapi-file: - name: Publish OpenAPI file - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - name: Generate OpenAPI file - run: | - cd crates/openapi-generator - cargo run --release -- --pretty --output ../../meilisearch.json - - name: Upload OpenAPI to Release - # No need to upload for dry run (cron) - if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.2 - with: - repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} - file: ./meilisearch.json - asset_name: meilisearch-openapi.json - tag: ${{ github.ref }} + publish-openapi-file: + name: Publish OpenAPI file + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Generate OpenAPI file + run: | + cd crates/openapi-generator + cargo run --release -- --pretty --output ../../meilisearch.json + - name: Upload OpenAPI to Release + # No need to upload for dry run (cron) + if: github.event_name == 'release' + uses: svenstaro/upload-release-action@2.11.2 + with: + repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} + file: ./meilisearch.json + asset_name: meilisearch-openapi.json + tag: ${{ github.ref }} From de52fe91f5a1b815c90ea4017f2504f964bf75a0 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 14:52:50 +0200 Subject: [PATCH 58/76] Update version --- Cargo.lock | 34 +++++++++++++++++----------------- Cargo.toml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6894e4856..28a51d495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "bumpalo", @@ -770,7 +770,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "time", @@ -1774,7 +1774,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "big_s", @@ -2006,7 +2006,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.17.0" +version = "1.17.1" dependencies = [ "tempfile", "thiserror 2.0.12", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.17.0" +version = "1.17.1" dependencies = [ "insta", "nom", @@ -2049,7 +2049,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.17.0" +version = "1.17.1" dependencies = [ "criterion", "serde_json", @@ -2194,7 +2194,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.17.0" +version = "1.17.1" dependencies = [ "arbitrary", "bumpalo", @@ -2994,7 +2994,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "backoff", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.17.0" +version = "1.17.1" dependencies = [ "criterion", "serde_json", @@ -3724,7 +3724,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.17.0" +version = "1.17.1" dependencies = [ "insta", "md5", @@ -3735,7 +3735,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.17.0" +version = "1.17.1" dependencies = [ "actix-cors", "actix-http", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.17.0" +version = "1.17.1" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.17.0" +version = "1.17.1" dependencies = [ "actix-web", "anyhow", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "clap", @@ -3919,7 +3919,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.17.0" +version = "1.17.1" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4471,7 +4471,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.17.0" +version = "1.17.1" dependencies = [ "big_s", "serde_json", @@ -7259,7 +7259,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index 1fa86c671..9708aebbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ ] [workspace.package] -version = "1.17.0" +version = "1.17.1" authors = [ "Quentin de Quelen ", "Clément Renault ", From 9021cb4258951341f5fcc747abb81f6aaf38ecc5 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 14:55:57 +0200 Subject: [PATCH 59/76] Fix update-cargo-version CI --- .github/workflows/update-cargo-toml-version.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-cargo-toml-version.yml b/.github/workflows/update-cargo-toml-version.yml index d13a4404a..4118cd651 100644 --- a/.github/workflows/update-cargo-toml-version.yml +++ b/.github/workflows/update-cargo-toml-version.yml @@ -41,5 +41,4 @@ jobs: --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \ --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \ --label 'skip changelog' \ - --milestone $NEW_VERSION \ --base $GITHUB_REF_NAME From b9e014c04496a21c2ea188ab37c47071394a0517 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 15:36:28 +0200 Subject: [PATCH 60/76] Update snapshots --- .../upgrade_failure/after_processing_everything.snap | 4 ++-- .../upgrade_failure/register_automatic_upgrade_task.snap | 2 +- .../registered_a_task_while_the_upgrade_task_is_enqueued.snap | 2 +- .../test_failure.rs/upgrade_failure/upgrade_task_failed.snap | 4 ++-- .../upgrade_failure/upgrade_task_failed_again.snap | 4 ++-- .../upgrade_failure/upgrade_task_succeeded.snap | 4 ++-- crates/meilisearch/tests/upgrade/mod.rs | 4 ++-- ...ches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ..._whole_batch_queue_once_everything_has_been_processed.snap | 2 +- ...e_whole_task_queue_once_everything_has_been_processed.snap | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 0b87ffa2c..d700dd3db 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index d0444d3ab..ee3cefba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 0533b162a..abaffbb1b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 8a52efde5..9569ecfe3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 40faa3504..1d7945023 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 8fe59f5c4..869d1d0b2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index 5585e50c4..5d120ba2f 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.1"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.17.1 is higher than the Meilisearch version 1.17.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.17.2 is higher than the Meilisearch version 1.17.1. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 842e3f536..8103ceed2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 18d5b4e20..81259377c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", From 0f1c78b185a665475701ef28806006691b2994de Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Fri, 1 Aug 2025 23:17:19 +0200 Subject: [PATCH 61/76] Add index rename feature --- .../index-scheduler/src/index_mapper/mod.rs | 14 ++++++++ crates/index-scheduler/src/processing.rs | 6 ++++ .../src/scheduler/autobatcher.rs | 14 +++++++- .../src/scheduler/create_batch.rs | 20 ++++++++++- .../src/scheduler/process_batch.rs | 16 ++++++++- crates/meilisearch-types/src/tasks.rs | 34 ++++++++++++++++++- 6 files changed, 100 insertions(+), 4 deletions(-) diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index e6bdccd41..d578b03dd 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -526,6 +526,20 @@ impl IndexMapper { Ok(()) } + /// Rename an index. + pub fn rename(&self, wtxn: &mut RwTxn, current: &str, new: &str) -> Result<()> { + let uuid = self + .index_mapping + .get(wtxn, current)? + .ok_or_else(|| Error::IndexNotFound(current.to_string()))?; + if self.index_mapping.get(wtxn, new)?.is_some() { + return Err(Error::IndexAlreadyExists(new.to_string())); + } + self.index_mapping.delete(wtxn, current)?; + self.index_mapping.put(wtxn, new, &uuid)?; + Ok(()) + } + /// The stats of an index. /// /// If available in the cache, they are directly returned. diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index 3da81f143..84b0a5360 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -125,6 +125,12 @@ make_enum_progress! { } } +make_enum_progress! { + pub enum RenameIndexProgress { + RenamingTheIndex, + } +} + make_enum_progress! { pub enum DeleteIndexProgress { DeletingTheIndex, diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index b3f7d2743..6b62d36e0 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -24,6 +24,7 @@ enum AutobatchKind { IndexCreation, IndexDeletion, IndexUpdate, + IndexRename, IndexSwap, } @@ -67,6 +68,7 @@ impl From for AutobatchKind { KindWithContent::IndexDeletion { .. } => AutobatchKind::IndexDeletion, KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation, KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate, + KindWithContent::IndexRename { .. } => AutobatchKind::IndexRename, KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap, KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } @@ -115,6 +117,9 @@ pub enum BatchKind { IndexUpdate { id: TaskId, }, + IndexRename { + id: TaskId, + }, IndexSwap { id: TaskId, }, @@ -176,6 +181,13 @@ impl BatchKind { )), false, ), + K::IndexRename => ( + Break(( + BatchKind::IndexRename { id: task_id }, + BatchStopReason::TaskCannotBeBatched { kind, id: task_id }, + )), + false, + ), K::IndexSwap => ( Break(( BatchKind::IndexSwap { id: task_id }, @@ -288,7 +300,7 @@ impl BatchKind { match (self, autobatch_kind) { // We don't batch any of these operations - (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), + (this, K::IndexCreation | K::IndexUpdate | K::IndexRename | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists. (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => { Break((this, BatchStopReason::IndexCreationMismatch { id })) diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index e78ed2c2e..fa84200a1 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -40,6 +40,11 @@ pub(crate) enum Batch { primary_key: Option, task: Task, }, + IndexRename { + index_uid: String, + new_index_uid: String, + task: Task, + }, IndexDeletion { index_uid: String, tasks: Vec, @@ -108,7 +113,8 @@ impl Batch { | Batch::Dump(task) | Batch::IndexCreation { task, .. } | Batch::Export { task } - | Batch::IndexUpdate { task, .. } => { + | Batch::IndexUpdate { task, .. } + | Batch::IndexRename { task, .. } => { RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap() } Batch::SnapshotCreation(tasks) @@ -153,6 +159,7 @@ impl Batch { IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } + | IndexRename { index_uid, .. } | IndexDeletion { index_uid, .. } => Some(index_uid), } } @@ -171,6 +178,7 @@ impl fmt::Display for Batch { Batch::IndexOperation { op, .. } => write!(f, "{op}")?, Batch::IndexCreation { .. } => f.write_str("IndexCreation")?, Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?, + Batch::IndexRename { .. } => f.write_str("IndexRename")?, Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?, Batch::IndexSwap { .. } => f.write_str("IndexSwap")?, Batch::Export { .. } => f.write_str("Export")?, @@ -411,6 +419,16 @@ impl IndexScheduler { }; Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task })) } + BatchKind::IndexRename { id } => { + let mut task = + self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; + current_batch.processing(Some(&mut task)); + let (new_uid) = match &task.kind { + KindWithContent::IndexRename { new_index_uid, .. } => new_index_uid.clone(), + _ => unreachable!(), + }; + Ok(Some(Batch::IndexRename { index_uid, new_index_uid: new_uid, task })) + } BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion { index_uid, index_has_been_created: must_create_index, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index c21ab27ad..d801287a7 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -15,7 +15,7 @@ use super::create_batch::Batch; use crate::processing::{ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, - UpdateIndexProgress, + UpdateIndexProgress, RenameIndexProgress, }; use crate::utils::{ self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task, @@ -229,6 +229,20 @@ impl IndexScheduler { progress, ) } + Batch::IndexRename { index_uid, new_index_uid, mut task } => { + progress.update_progress(RenameIndexProgress::RenamingTheIndex); + let mut wtxn = self.env.write_txn()?; + self.index_mapper.rename(&mut wtxn, &index_uid, &new_index_uid)?; + self.queue.tasks.update_index(&mut wtxn, &new_index_uid, |bm| { + let old = self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); + *bm |= &old; + })?; + self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; + wtxn.commit()?; + task.status = Status::Succeeded; + task.details = Some(Details::IndexRename(IndexRenameDetails { old_uid: index_uid, new_uid: new_index_uid })); + Ok((vec![task], ProcessBatchInfo::default())) + } Batch::IndexUpdate { index_uid, primary_key, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); let rtxn = self.env.read_txn()?; diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 99b04f1e3..64826c693 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -141,6 +141,10 @@ pub enum KindWithContent { index_uid: String, primary_key: Option, }, + IndexRename { + index_uid: String, + new_index_uid: String, + }, IndexSwap { swaps: Vec, }, @@ -174,6 +178,13 @@ pub struct IndexSwap { pub indexes: (String, String), } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct IndexRenameDetails { + pub old_uid: String, + pub new_uid: String, +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct ExportIndexSettings { @@ -193,6 +204,7 @@ impl KindWithContent { KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, + KindWithContent::IndexRename { .. } => Kind::IndexRename, KindWithContent::IndexSwap { .. } => Kind::IndexSwap, KindWithContent::TaskCancelation { .. } => Kind::TaskCancelation, KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, @@ -222,6 +234,7 @@ impl KindWithContent { | IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } | IndexDeletion { index_uid } => vec![index_uid], + IndexRename { index_uid, new_index_uid } => vec![index_uid, new_index_uid], IndexSwap { swaps } => { let mut indexes = HashSet::<&str>::default(); for swap in swaps { @@ -274,6 +287,12 @@ impl KindWithContent { | KindWithContent::IndexUpdate { primary_key, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone() }) } + KindWithContent::IndexRename { index_uid, new_index_uid } => { + Some(Details::IndexRename { + old_uid: index_uid.clone(), + new_uid: new_index_uid.clone(), + }) + } KindWithContent::IndexSwap { swaps } => { Some(Details::IndexSwap { swaps: swaps.clone() }) } @@ -344,6 +363,12 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, + KindWithContent::IndexRename { index_uid, new_index_uid } => { + Some(Details::IndexRename { + old_uid: index_uid.clone(), + new_uid: new_index_uid.clone(), + }) + } KindWithContent::IndexCreation { primary_key, .. } | KindWithContent::IndexUpdate { primary_key, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone() }) @@ -538,6 +563,7 @@ pub enum Kind { IndexCreation, IndexDeletion, IndexUpdate, + IndexRename, IndexSwap, TaskCancelation, TaskDeletion, @@ -556,7 +582,8 @@ impl Kind { | Kind::SettingsUpdate | Kind::IndexCreation | Kind::IndexDeletion - | Kind::IndexUpdate => true, + | Kind::IndexUpdate + | Kind::IndexRename => true, Kind::IndexSwap | Kind::TaskCancelation | Kind::TaskDeletion @@ -577,6 +604,7 @@ impl Display for Kind { Kind::IndexCreation => write!(f, "indexCreation"), Kind::IndexDeletion => write!(f, "indexDeletion"), Kind::IndexUpdate => write!(f, "indexUpdate"), + Kind::IndexRename => write!(f, "indexRename"), Kind::IndexSwap => write!(f, "indexSwap"), Kind::TaskCancelation => write!(f, "taskCancelation"), Kind::TaskDeletion => write!(f, "taskDeletion"), @@ -595,6 +623,8 @@ impl FromStr for Kind { Ok(Kind::IndexCreation) } else if kind.eq_ignore_ascii_case("indexUpdate") { Ok(Kind::IndexUpdate) + } else if kind.eq_ignore_ascii_case("indexRename") { + Ok(Kind::IndexRename) } else if kind.eq_ignore_ascii_case("indexSwap") { Ok(Kind::IndexSwap) } else if kind.eq_ignore_ascii_case("indexDeletion") { @@ -692,6 +722,7 @@ pub enum Details { IndexSwap { swaps: Vec, }, + IndexRename(IndexRenameDetails), Export { url: String, api_key: Option, @@ -737,6 +768,7 @@ impl Details { Self::SettingsUpdate { .. } | Self::IndexInfo { .. } | Self::Dump { .. } + | Self::IndexRename { .. } | Self::Export { .. } | Self::UpgradeDatabase { .. } | Self::IndexSwap { .. } => (), From ae2d0a67a4f91bb95c40e721b2b486dfce22400a Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 5 Aug 2025 19:18:05 +0200 Subject: [PATCH 62/76] Enhance index update functionality to support renaming by adding new_uid field. Update related structures and methods to handle the new index UID during updates, ensuring backward compatibility with existing index operations. --- crates/dump/src/lib.rs | 5 +- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +- crates/index-scheduler/src/dump.rs | 3 +- crates/index-scheduler/src/insta_snapshot.rs | 4 +- crates/index-scheduler/src/processing.rs | 6 - .../src/scheduler/autobatcher.rs | 16 +- .../src/scheduler/autobatcher_test.rs | 6 +- .../src/scheduler/create_batch.rs | 29 +- .../src/scheduler/process_batch.rs | 62 +-- crates/index-scheduler/src/utils.rs | 11 +- crates/meilisearch-types/src/task_view.rs | 25 +- crates/meilisearch-types/src/tasks.rs | 65 ++- crates/meilisearch/src/routes/indexes/mod.rs | 10 + crates/meilisearch/tests/index/mod.rs | 1 + .../meilisearch/tests/index/rename_index.rs | 419 ++++++++++++++++++ 15 files changed, 547 insertions(+), 119 deletions(-) create mode 100644 crates/meilisearch/tests/index/rename_index.rs diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index 81ba40944..b4b339f09 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -129,6 +129,7 @@ pub enum KindDump { }, IndexUpdate { primary_key: Option, + new_uid: Option, }, IndexSwap { swaps: Vec, @@ -210,8 +211,8 @@ impl From for KindDump { KindWithContent::IndexCreation { primary_key, .. } => { KindDump::IndexCreation { primary_key } } - KindWithContent::IndexUpdate { primary_key, .. } => { - KindDump::IndexUpdate { primary_key } + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindDump::IndexUpdate { primary_key, new_uid: new_index_uid } } KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps }, KindWithContent::TaskCancelation { query, tasks } => { diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 3a0c8ef0d..790c239d7 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -85,7 +85,7 @@ impl CompatV5ToV6 { v6::Kind::IndexCreation { primary_key } } v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => { - v6::Kind::IndexUpdate { primary_key } + v6::Kind::IndexUpdate { primary_key, new_uid: None } } v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion, v5::tasks::TaskContent::DocumentAddition { @@ -141,7 +141,7 @@ impl CompatV5ToV6 { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key } + v6::Details::IndexInfo { primary_key, new_uid: None } } v5::Details::DocumentDeletion { received_document_ids, diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 1e681c8e8..18c665ca3 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -197,9 +197,10 @@ impl<'a> Dump<'a> { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, }, - KindDump::IndexUpdate { primary_key } => KindWithContent::IndexUpdate { + KindDump::IndexUpdate { primary_key, new_uid } => KindWithContent::IndexUpdate { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, + new_index_uid: new_uid, }, KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps }, KindDump::TaskCancelation { query, tasks } => { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index cb804d9b4..6d72e4b9f 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key } => { - format!("{{ primary_key: {primary_key:?} }}") + Details::IndexInfo { primary_key, new_uid } => { + format!("{{ primary_key: {primary_key:?}, new_uid: {new_uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index 84b0a5360..3da81f143 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -125,12 +125,6 @@ make_enum_progress! { } } -make_enum_progress! { - pub enum RenameIndexProgress { - RenamingTheIndex, - } -} - make_enum_progress! { pub enum DeleteIndexProgress { DeletingTheIndex, diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index 6b62d36e0..a88a9f0bf 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -24,7 +24,6 @@ enum AutobatchKind { IndexCreation, IndexDeletion, IndexUpdate, - IndexRename, IndexSwap, } @@ -68,7 +67,6 @@ impl From for AutobatchKind { KindWithContent::IndexDeletion { .. } => AutobatchKind::IndexDeletion, KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation, KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate, - KindWithContent::IndexRename { .. } => AutobatchKind::IndexRename, KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap, KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } @@ -117,9 +115,6 @@ pub enum BatchKind { IndexUpdate { id: TaskId, }, - IndexRename { - id: TaskId, - }, IndexSwap { id: TaskId, }, @@ -181,13 +176,6 @@ impl BatchKind { )), false, ), - K::IndexRename => ( - Break(( - BatchKind::IndexRename { id: task_id }, - BatchStopReason::TaskCannotBeBatched { kind, id: task_id }, - )), - false, - ), K::IndexSwap => ( Break(( BatchKind::IndexSwap { id: task_id }, @@ -299,8 +287,8 @@ impl BatchKind { }; match (self, autobatch_kind) { - // We don't batch any of these operations - (this, K::IndexCreation | K::IndexUpdate | K::IndexRename | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), + // We don't batch any of these operations + (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists. (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => { Break((this, BatchStopReason::IndexCreationMismatch { id })) diff --git a/crates/index-scheduler/src/scheduler/autobatcher_test.rs b/crates/index-scheduler/src/scheduler/autobatcher_test.rs index 435ce55c6..0653753dc 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher_test.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher_test.rs @@ -75,7 +75,11 @@ fn idx_create() -> KindWithContent { } fn idx_update() -> KindWithContent { - KindWithContent::IndexUpdate { index_uid: String::from("doggo"), primary_key: None } + KindWithContent::IndexUpdate { + index_uid: String::from("doggo"), + primary_key: None, + new_index_uid: None, + } } fn idx_del() -> KindWithContent { diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index fa84200a1..14aa307cd 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -38,11 +38,7 @@ pub(crate) enum Batch { IndexUpdate { index_uid: String, primary_key: Option, - task: Task, - }, - IndexRename { - index_uid: String, - new_index_uid: String, + new_index_uid: Option, task: Task, }, IndexDeletion { @@ -113,8 +109,7 @@ impl Batch { | Batch::Dump(task) | Batch::IndexCreation { task, .. } | Batch::Export { task } - | Batch::IndexUpdate { task, .. } - | Batch::IndexRename { task, .. } => { + | Batch::IndexUpdate { task, .. } => { RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap() } Batch::SnapshotCreation(tasks) @@ -159,7 +154,6 @@ impl Batch { IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } - | IndexRename { index_uid, .. } | IndexDeletion { index_uid, .. } => Some(index_uid), } } @@ -178,7 +172,6 @@ impl fmt::Display for Batch { Batch::IndexOperation { op, .. } => write!(f, "{op}")?, Batch::IndexCreation { .. } => f.write_str("IndexCreation")?, Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?, - Batch::IndexRename { .. } => f.write_str("IndexRename")?, Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?, Batch::IndexSwap { .. } => f.write_str("IndexSwap")?, Batch::Export { .. } => f.write_str("Export")?, @@ -413,21 +406,13 @@ impl IndexScheduler { let mut task = self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; current_batch.processing(Some(&mut task)); - let primary_key = match &task.kind { - KindWithContent::IndexUpdate { primary_key, .. } => primary_key.clone(), + let (primary_key, new_index_uid) = match &task.kind { + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + (primary_key.clone(), new_index_uid.clone()) + } _ => unreachable!(), }; - Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task })) - } - BatchKind::IndexRename { id } => { - let mut task = - self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; - current_batch.processing(Some(&mut task)); - let (new_uid) = match &task.kind { - KindWithContent::IndexRename { new_index_uid, .. } => new_index_uid.clone(), - _ => unreachable!(), - }; - Ok(Some(Batch::IndexRename { index_uid, new_index_uid: new_uid, task })) + Ok(Some(Batch::IndexUpdate { index_uid, primary_key, new_index_uid, task })) } BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion { index_uid, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index d801287a7..a82324fc1 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -15,7 +15,7 @@ use super::create_batch::Batch; use crate::processing::{ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, - UpdateIndexProgress, RenameIndexProgress, + UpdateIndexProgress, }; use crate::utils::{ self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task, @@ -224,38 +224,47 @@ impl IndexScheduler { self.index_mapper.create_index(wtxn, &index_uid, None)?; self.process_batch( - Batch::IndexUpdate { index_uid, primary_key, task }, + Batch::IndexUpdate { index_uid, primary_key, new_index_uid: None, task }, current_batch, progress, ) } - Batch::IndexRename { index_uid, new_index_uid, mut task } => { - progress.update_progress(RenameIndexProgress::RenamingTheIndex); - let mut wtxn = self.env.write_txn()?; - self.index_mapper.rename(&mut wtxn, &index_uid, &new_index_uid)?; - self.queue.tasks.update_index(&mut wtxn, &new_index_uid, |bm| { - let old = self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); - *bm |= &old; - })?; - self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; - wtxn.commit()?; - task.status = Status::Succeeded; - task.details = Some(Details::IndexRename(IndexRenameDetails { old_uid: index_uid, new_uid: new_index_uid })); - Ok((vec![task], ProcessBatchInfo::default())) - } - Batch::IndexUpdate { index_uid, primary_key, mut task } => { + Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); - let rtxn = self.env.read_txn()?; - let index = self.index_mapper.index(&rtxn, &index_uid)?; - if let Some(primary_key) = primary_key.clone() { + // Handle rename if new_index_uid is provided + let final_index_uid = if let Some(new_uid) = &new_index_uid { + let mut wtxn = self.env.write_txn()?; + + // Rename the index + self.index_mapper.rename(&mut wtxn, &index_uid, new_uid)?; + + // Update the task index mappings + let old_tasks = + self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); + self.queue.tasks.update_index(&mut wtxn, new_uid, |bm| { + *bm |= &old_tasks; + })?; + self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; + wtxn.commit()?; + + new_uid.clone() + } else { + index_uid.clone() + }; + // Get the index (renamed or not) + let rtxn = self.env.read_txn()?; + let index = self.index_mapper.index(&rtxn, &final_index_uid)?; + + // Handle primary key update if provided + if let Some(ref primary_key) = primary_key { let mut index_wtxn = index.write_txn()?; let mut builder = MilliSettings::new( &mut index_wtxn, &index, self.index_mapper.indexer_config(), ); - builder.set_primary_key(primary_key); + builder.set_primary_key(primary_key.clone()); let must_stop_processing = self.scheduler.must_stop_processing.clone(); builder @@ -264,7 +273,7 @@ impl IndexScheduler { &progress, current_batch.embedder_stats.clone(), ) - .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?; index_wtxn.commit()?; } @@ -272,7 +281,10 @@ impl IndexScheduler { rtxn.commit()?; task.status = Status::Succeeded; - task.details = Some(Details::IndexInfo { primary_key }); + task.details = Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_uid: new_index_uid.clone(), + }); // if the update processed successfully, we're going to store the new // stats of the index. Since the tasks have already been processed and @@ -282,8 +294,8 @@ impl IndexScheduler { let mut wtxn = self.env.write_txn()?; let index_rtxn = index.read_txn()?; let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn) - .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; - self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.clone())))?; + self.index_mapper.store_stats_of(&mut wtxn, &final_index_uid, &stats)?; wtxn.commit()?; Ok(()) }(); diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 3c921f099..91bba35d7 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -264,7 +264,12 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid), K::IndexDeletion { index_uid } => index_uids.push(index_uid), K::IndexCreation { index_uid, .. } => index_uids.push(index_uid), - K::IndexUpdate { index_uid, .. } => index_uids.push(index_uid), + K::IndexUpdate { index_uid, new_index_uid, .. } => { + index_uids.push(index_uid); + if let Some(new_uid) = new_index_uid { + index_uids.push(new_uid); + } + } K::IndexSwap { swaps } => { for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { @@ -496,9 +501,9 @@ impl crate::IndexScheduler { Details::SettingsUpdate { settings: _ } => { assert_eq!(kind.as_kind(), Kind::SettingsUpdate); } - Details::IndexInfo { primary_key: pk1 } => match &kind { + Details::IndexInfo { primary_key: pk1, .. } => match &kind { KindWithContent::IndexCreation { index_uid, primary_key: pk2 } - | KindWithContent::IndexUpdate { index_uid, primary_key: pk2 } => { + | KindWithContent::IndexUpdate { index_uid, primary_key: pk2, .. } => { self.queue .tasks .index_tasks diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 7521137c0..460ae68d7 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -132,6 +132,11 @@ pub struct DetailsView { pub payload_size: Option, #[serde(skip_serializing_if = "Option::is_none")] pub indexes: Option>, + // index rename + #[serde(skip_serializing_if = "Option::is_none")] + pub old_index_uid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_index_uid: Option, } impl DetailsView { @@ -292,6 +297,18 @@ impl DetailsView { (None, Some(to)) | (Some(to), None) => Some(to), (Some(_), Some(to)) => Some(to), }, + old_index_uid: match (self.old_index_uid.clone(), other.old_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, + new_index_uid: match (self.new_index_uid.clone(), other.new_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, } } } @@ -324,9 +341,11 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key } => { - DetailsView { primary_key: Some(primary_key), ..DetailsView::default() } - } + Details::IndexInfo { primary_key, new_uid } => DetailsView { + primary_key: Some(primary_key), + new_index_uid: new_uid.clone(), + ..DetailsView::default() + }, Details::DocumentDeletion { provided_ids: received_document_ids, deleted_documents, diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 64826c693..f7b59b299 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -140,10 +140,7 @@ pub enum KindWithContent { IndexUpdate { index_uid: String, primary_key: Option, - }, - IndexRename { - index_uid: String, - new_index_uid: String, + new_index_uid: Option, }, IndexSwap { swaps: Vec, @@ -178,13 +175,6 @@ pub struct IndexSwap { pub indexes: (String, String), } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct IndexRenameDetails { - pub old_uid: String, - pub new_uid: String, -} - #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct ExportIndexSettings { @@ -204,7 +194,6 @@ impl KindWithContent { KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, - KindWithContent::IndexRename { .. } => Kind::IndexRename, KindWithContent::IndexSwap { .. } => Kind::IndexSwap, KindWithContent::TaskCancelation { .. } => Kind::TaskCancelation, KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, @@ -232,9 +221,14 @@ impl KindWithContent { | DocumentClear { index_uid } | SettingsUpdate { index_uid, .. } | IndexCreation { index_uid, .. } - | IndexUpdate { index_uid, .. } | IndexDeletion { index_uid } => vec![index_uid], - IndexRename { index_uid, new_index_uid } => vec![index_uid, new_index_uid], + IndexUpdate { index_uid, new_index_uid, .. } => { + let mut indexes = vec![index_uid.as_str()]; + if let Some(new_uid) = new_index_uid { + indexes.push(new_uid.as_str()); + } + indexes + } IndexSwap { swaps } => { let mut indexes = HashSet::<&str>::default(); for swap in swaps { @@ -283,13 +277,12 @@ impl KindWithContent { KindWithContent::SettingsUpdate { new_settings, .. } => { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexCreation { primary_key, .. } => { + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) } - KindWithContent::IndexRename { index_uid, new_index_uid } => { - Some(Details::IndexRename { - old_uid: index_uid.clone(), + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), new_uid: new_index_uid.clone(), }) } @@ -363,16 +356,15 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexRename { index_uid, new_index_uid } => { - Some(Details::IndexRename { - old_uid: index_uid.clone(), + KindWithContent::IndexCreation { primary_key, .. } => { + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + } + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), new_uid: new_index_uid.clone(), }) } - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) - } KindWithContent::IndexSwap { .. } => { todo!() } @@ -426,10 +418,13 @@ impl From<&KindWithContent> for Option
{ } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) } - KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_uid: new_index_uid.clone(), + }) } KindWithContent::IndexSwap { .. } => None, KindWithContent::TaskCancelation { query, tasks } => Some(Details::TaskCancelation { @@ -563,7 +558,6 @@ pub enum Kind { IndexCreation, IndexDeletion, IndexUpdate, - IndexRename, IndexSwap, TaskCancelation, TaskDeletion, @@ -582,8 +576,7 @@ impl Kind { | Kind::SettingsUpdate | Kind::IndexCreation | Kind::IndexDeletion - | Kind::IndexUpdate - | Kind::IndexRename => true, + | Kind::IndexUpdate => true, Kind::IndexSwap | Kind::TaskCancelation | Kind::TaskDeletion @@ -604,7 +597,6 @@ impl Display for Kind { Kind::IndexCreation => write!(f, "indexCreation"), Kind::IndexDeletion => write!(f, "indexDeletion"), Kind::IndexUpdate => write!(f, "indexUpdate"), - Kind::IndexRename => write!(f, "indexRename"), Kind::IndexSwap => write!(f, "indexSwap"), Kind::TaskCancelation => write!(f, "taskCancelation"), Kind::TaskDeletion => write!(f, "taskDeletion"), @@ -623,8 +615,6 @@ impl FromStr for Kind { Ok(Kind::IndexCreation) } else if kind.eq_ignore_ascii_case("indexUpdate") { Ok(Kind::IndexUpdate) - } else if kind.eq_ignore_ascii_case("indexRename") { - Ok(Kind::IndexRename) } else if kind.eq_ignore_ascii_case("indexSwap") { Ok(Kind::IndexSwap) } else if kind.eq_ignore_ascii_case("indexDeletion") { @@ -687,6 +677,7 @@ pub enum Details { }, IndexInfo { primary_key: Option, + new_uid: Option, }, DocumentDeletion { provided_ids: usize, @@ -722,7 +713,6 @@ pub enum Details { IndexSwap { swaps: Vec, }, - IndexRename(IndexRenameDetails), Export { url: String, api_key: Option, @@ -768,7 +758,6 @@ impl Details { Self::SettingsUpdate { .. } | Self::IndexInfo { .. } | Self::Dump { .. } - | Self::IndexRename { .. } | Self::Export { .. } | Self::UpgradeDatabase { .. } | Self::IndexSwap { .. } => (), diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 04b3e12c4..632922542 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -375,6 +375,9 @@ pub struct UpdateIndexRequest { /// The new primary key of the index #[deserr(default, error = DeserrJsonError)] primary_key: Option, + /// The new uid of the index (for renaming) + #[deserr(default, error = DeserrJsonError)] + uid: Option, } /// Update index @@ -419,6 +422,12 @@ pub async fn update_index( debug!(parameters = ?body, "Update index"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let body = body.into_inner(); + + // Validate new uid if provided + if let Some(ref new_uid) = body.uid { + let _ = IndexUid::try_from(new_uid.clone())?; + } + analytics.publish( IndexUpdatedAggregate { primary_key: body.primary_key.iter().cloned().collect() }, &req, @@ -427,6 +436,7 @@ pub async fn update_index( let task = KindWithContent::IndexUpdate { index_uid: index_uid.into_inner(), primary_key: body.primary_key, + new_index_uid: body.uid, }; let uid = get_task_id(&req, &opt)?; diff --git a/crates/meilisearch/tests/index/mod.rs b/crates/meilisearch/tests/index/mod.rs index 5df5e7e97..f3cb7fde9 100644 --- a/crates/meilisearch/tests/index/mod.rs +++ b/crates/meilisearch/tests/index/mod.rs @@ -2,5 +2,6 @@ mod create_index; mod delete_index; mod errors; mod get_index; +mod rename_index; mod stats; mod update_index; diff --git a/crates/meilisearch/tests/index/rename_index.rs b/crates/meilisearch/tests/index/rename_index.rs new file mode 100644 index 000000000..793b079f1 --- /dev/null +++ b/crates/meilisearch/tests/index/rename_index.rs @@ -0,0 +1,419 @@ +use crate::common::{shared_does_not_exists_index, Server}; +use crate::json; + +#[actix_rt::test] +async fn rename_index_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename via PATCH update endpoint + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); +} + +#[actix_rt::test] +async fn rename_to_existing_index_via_patch() { + let server = Server::new_shared(); + let index1 = server.unique_index(); + let index2 = server.unique_index(); + + // Create both indexes + let (task, code) = index1.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let (task, code) = index2.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename index1 to index2's uid via PATCH (should fail) + let body = json!({ "uid": index2.uid }); + let (task, code) = index1.service.patch(format!("/indexes/{}", index1.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` already exists.", index2.uid), + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_non_existent_index_via_patch() { + let server = Server::new_shared(); + let index = shared_does_not_exists_index().await; + + // Try to rename non-existent index via PATCH + let body = json!({ "uid": "new_name" }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` not found.", index.uid), + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_with_invalid_uid_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename with invalid uid via PATCH + let body = json!({ "uid": "Invalid UID!" }); + let (_, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn rename_index_with_documents_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1" }, + { "id": 2, "title": "Movie 2" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index via PATCH + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify documents are accessible in renamed index + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn rename_index_and_update_primary_key_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index without primary key + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename index and set primary key at the same time + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ + "uid": &new_uid, + "primaryKey": "id" + }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded and primary key was set + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + assert_eq!(response["details"]["primaryKey"], "id"); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists with correct primary key + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); + assert_eq!(response["primaryKey"], "id"); +} + +#[actix_rt::test] +async fn rename_index_and_verify_stats() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1", "genre": "Action" }, + { "id": 2, "title": "Movie 2", "genre": "Drama" }, + { "id": 3, "title": "Movie 3", "genre": "Comedy" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats before rename + let (stats_before, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!(stats_before["numberOfDocuments"], 3); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats after rename using the new uid + let (stats_after, code) = server.service.get(format!("/indexes/{}/stats", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(stats_after["numberOfDocuments"], 3); + assert_eq!(stats_after["numberOfDocuments"], stats_before["numberOfDocuments"]); +} + +#[actix_rt::test] +async fn rename_index_preserves_settings() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Configure settings + let settings = json!({ + "searchableAttributes": ["title", "description"], + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": ["the", "a", "an"], + "synonyms": { + "movie": ["film", "picture"], + "great": ["awesome", "excellent"] + } + }); + + let (task, code) = index.update_settings(settings.clone()).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify settings are preserved + let (settings_after, code) = server.service.get(format!("/indexes/{}/settings", new_uid)).await; + assert_eq!(code, 200); + + assert_eq!(settings_after["searchableAttributes"], json!(["title", "description"])); + assert_eq!(settings_after["filterableAttributes"], json!(["genre", "year"])); + assert_eq!(settings_after["sortableAttributes"], json!(["year"])); + + // Check stopWords contains the same items (order may vary) + let stop_words = settings_after["stopWords"].as_array().unwrap(); + assert_eq!(stop_words.len(), 3); + assert!(stop_words.contains(&json!("the"))); + assert!(stop_words.contains(&json!("a"))); + assert!(stop_words.contains(&json!("an"))); + + assert_eq!(settings_after["synonyms"]["movie"], json!(["film", "picture"])); + assert_eq!(settings_after["synonyms"]["great"], json!(["awesome", "excellent"])); +} + +#[actix_rt::test] +async fn rename_index_preserves_search_functionality() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "The Matrix", "genre": "Sci-Fi", "year": 1999 }, + { "id": 2, "title": "Inception", "genre": "Sci-Fi", "year": 2010 }, + { "id": 3, "title": "The Dark Knight", "genre": "Action", "year": 2008 } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Make settings filterable + let settings = json!({ + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"] + }); + let (task, code) = index.update_settings(settings).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search before rename + let search_params = json!({ + "q": "matrix", + "filter": "genre = 'Sci-Fi'", + "sort": ["year:asc"] + }); + let (results_before, code) = index.search_post(search_params.clone()).await; + assert_eq!(code, 200); + assert_eq!(results_before["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_before["hits"][0]["title"], "The Matrix"); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search after rename + let (results_after, code) = + server.service.post(format!("/indexes/{}/search", new_uid), search_params).await; + assert_eq!(code, 200); + assert_eq!(results_after["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_after["hits"][0]["title"], "The Matrix"); + + // Verify facet search also works + let facet_search = json!({ + "facetQuery": "Sci", + "facetName": "genre" + }); + let (facet_results, code) = + server.service.post(format!("/indexes/{}/facet-search", new_uid), facet_search).await; + assert_eq!(code, 200); + assert_eq!(facet_results["facetHits"].as_array().unwrap().len(), 1); + assert_eq!(facet_results["facetHits"][0]["value"], "Sci-Fi"); +} + +#[actix_rt::test] +async fn rename_index_with_pending_tasks() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Add initial documents + let documents = json!([ + { "id": 1, "title": "Document 1" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Start a rename + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (rename_task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + assert_eq!(code, 202); + + // Try to add documents to the old index while rename is pending + let more_documents = json!([ + { "id": 2, "title": "Document 2" } + ]); + let (_, code) = index.add_documents(more_documents, None).await; + assert_eq!(code, 202); + + // Wait for rename to complete + server.wait_task(rename_task.uid()).await.succeeded(); + + // Add documents to the new index + let final_documents = json!([ + { "id": 3, "title": "Document 3" } + ]); + let (task, code) = + server.service.post(format!("/indexes/{}/documents", new_uid), final_documents).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify all documents are accessible + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + let docs = response["results"].as_array().unwrap(); + assert!(!docs.is_empty()); // At least the initial document should be there +} + +#[actix_rt::test] +async fn rename_index_to_same_name() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename to the same name + let body = json!({ "uid": index.uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + // Should fail with index already exists error + assert_eq!(response["status"], "failed"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["error"]["code"], "index_already_exists"); + + // Index should still be accessible with original name + let (_, code) = index.get().await; + assert_eq!(code, 200); +} From ae5bd9d0e338cceb121b3975fb0899e5d50bf102 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Aug 2025 17:34:11 +0200 Subject: [PATCH 63/76] fix: updated_at was not 'updated' when updating the index name --- .../src/scheduler/process_batch.rs | 14 +++++--- .../meilisearch/tests/index/update_index.rs | 33 +++++++++++++++++++ crates/milli/src/index.rs | 2 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index a82324fc1..beb7c021f 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -10,6 +10,7 @@ use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use milli::update::Settings as MilliSettings; use roaring::RoaringBitmap; +use time::OffsetDateTime; use super::create_batch::Batch; use crate::processing::{ @@ -232,8 +233,15 @@ impl IndexScheduler { Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); + // Get the index (renamed or not) + let rtxn = self.env.read_txn()?; + let index = self.index_mapper.index(&rtxn, &index_uid)?; + let mut index_wtxn = index.write_txn()?; + // Handle rename if new_index_uid is provided let final_index_uid = if let Some(new_uid) = &new_index_uid { + index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; + let mut wtxn = self.env.write_txn()?; // Rename the index @@ -252,13 +260,9 @@ impl IndexScheduler { } else { index_uid.clone() }; - // Get the index (renamed or not) - let rtxn = self.env.read_txn()?; - let index = self.index_mapper.index(&rtxn, &final_index_uid)?; // Handle primary key update if provided if let Some(ref primary_key) = primary_key { - let mut index_wtxn = index.write_txn()?; let mut builder = MilliSettings::new( &mut index_wtxn, &index, @@ -274,9 +278,9 @@ impl IndexScheduler { current_batch.embedder_stats.clone(), ) .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?; - index_wtxn.commit()?; } + index_wtxn.commit()?; // drop rtxn before starting a new wtxn on the same db rtxn.commit()?; diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 262324bcf..45f3ea420 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -1,3 +1,5 @@ +use meili_snap::snapshot; + use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; @@ -106,3 +108,34 @@ async fn error_update_unexisting_index() { assert_eq!(response["error"], expected_response); } + +#[actix_rt::test] +async fn update_index_name() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, code) = index.create(None).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let new_index = server.unique_index(); + let (task, _status_code) = index.update_raw(json!({ "uid": new_index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (response, code) = new_index.get().await; + + snapshot!(code, @"200 OK"); + + assert_eq!(response["uid"], new_index.uid); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + + let created_at = + OffsetDateTime::parse(response["createdAt"].as_str().unwrap(), &Rfc3339).unwrap(); + let updated_at = + OffsetDateTime::parse(response["updatedAt"].as_str().unwrap(), &Rfc3339).unwrap(); + assert!(created_at < updated_at, "{created_at} should be inferior to {updated_at}"); + + snapshot!(response["primaryKey"], @"null"); + snapshot!(response.as_object().unwrap().len(), @"4"); +} diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 9f32fdb04..6429dabbc 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1457,7 +1457,7 @@ impl Index { .0) } - pub(crate) fn set_updated_at( + pub fn set_updated_at( &self, wtxn: &mut RwTxn<'_>, time: &time::OffsetDateTime, From ecea247e5d3e30c2562e3714114a44f726e6d4bf Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Aug 2025 19:35:07 +0200 Subject: [PATCH 64/76] Provide a rename argument to the swap --- crates/index-scheduler/src/error.rs | 10 + .../index-scheduler/src/queue/batches_test.rs | 6 +- .../index-scheduler/src/queue/tasks_test.rs | 6 +- .../src/scheduler/autobatcher_test.rs | 5 +- .../src/scheduler/process_batch.rs | 53 +++- crates/index-scheduler/src/scheduler/test.rs | 16 +- crates/index-scheduler/src/utils.rs | 6 +- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch-types/src/tasks.rs | 1 + crates/meilisearch/src/routes/swap_indexes.rs | 19 +- .../meilisearch/tests/swap_indexes/errors.rs | 17 ++ crates/meilisearch/tests/swap_indexes/mod.rs | 252 ++++++++++++++++++ 12 files changed, 358 insertions(+), 34 deletions(-) diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 60669ff2d..742576c73 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -67,6 +67,8 @@ pub enum Error { SwapDuplicateIndexesFound(Vec), #[error("Index `{0}` not found.")] SwapIndexNotFound(String), + #[error("Index `{0}` found during a rename. Renaming doen't overwrite the other index name.")] + SwapIndexFoundDuringRename(String), #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")] NoSpaceLeftInTaskQueue, #[error( @@ -74,6 +76,10 @@ pub enum Error { .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesNotFound(Vec), + #[error("Index {} found during a rename. Renaming doen't overwrite the other index name.", + .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") + )] + SwapIndexesFoundDuringRename(Vec), #[error("Corrupted dump.")] CorruptedDump, #[error( @@ -203,6 +209,8 @@ impl Error { | Error::SwapIndexNotFound(_) | Error::NoSpaceLeftInTaskQueue | Error::SwapIndexesNotFound(_) + | Error::SwapIndexFoundDuringRename(_) + | Error::SwapIndexesFoundDuringRename(_) | Error::CorruptedDump | Error::InvalidTaskDate { .. } | Error::InvalidTaskUid { .. } @@ -271,6 +279,8 @@ impl ErrorCode for Error { Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound, Error::SwapIndexNotFound(_) => Code::IndexNotFound, Error::SwapIndexesNotFound(_) => Code::IndexNotFound, + Error::SwapIndexFoundDuringRename(_) => Code::IndexNotFound, + Error::SwapIndexesFoundDuringRename(_) => Code::IndexNotFound, Error::InvalidTaskDate { field, .. } => (*field).into(), Error::InvalidTaskUid { .. } => Code::InvalidTaskUids, Error::InvalidBatchUid { .. } => Code::InvalidBatchUids, diff --git a/crates/index-scheduler/src/queue/batches_test.rs b/crates/index-scheduler/src/queue/batches_test.rs index 782acb4b1..49dc0ba61 100644 --- a/crates/index-scheduler/src/queue/batches_test.rs +++ b/crates/index-scheduler/src/queue/batches_test.rs @@ -334,11 +334,11 @@ fn query_batches_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -442,7 +442,7 @@ fn query_batches_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/queue/tasks_test.rs b/crates/index-scheduler/src/queue/tasks_test.rs index d60d621d1..bfe26accf 100644 --- a/crates/index-scheduler/src/queue/tasks_test.rs +++ b/crates/index-scheduler/src/queue/tasks_test.rs @@ -304,11 +304,11 @@ fn query_tasks_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -399,7 +399,7 @@ fn query_tasks_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/scheduler/autobatcher_test.rs b/crates/index-scheduler/src/scheduler/autobatcher_test.rs index 0653753dc..7dd4536b7 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher_test.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher_test.rs @@ -88,7 +88,10 @@ fn idx_del() -> KindWithContent { fn idx_swap() -> KindWithContent { KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: (String::from("doggo"), String::from("catto")) }], + swaps: vec![IndexSwap { + indexes: (String::from("doggo"), String::from("catto")), + rename: false, + }], } } diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index beb7c021f..7688b1787 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -360,13 +360,18 @@ impl IndexScheduler { unreachable!() }; let mut not_found_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { - for index in [lhs, rhs] { - let index_exists = self.index_mapper.index_exists(&wtxn, index)?; - if !index_exists { - not_found_indexes.insert(index); - } + let mut found_indexes_but_should_not = BTreeSet::new(); + for IndexSwap { indexes: (lhs, rhs), rename } in swaps { + let index_exists = self.index_mapper.index_exists(&wtxn, lhs)?; + if !index_exists { + not_found_indexes.insert(lhs); } + let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?; + match (index_exists, rename) { + (true, true) => found_indexes_but_should_not.insert(rhs), + (false, false) => not_found_indexes.insert(rhs), + (true, false) | (false, true) => true, // random value we don't read it anyway + }; } if !not_found_indexes.is_empty() { if not_found_indexes.len() == 1 { @@ -379,6 +384,17 @@ impl IndexScheduler { )); } } + if !found_indexes_but_should_not.is_empty() { + if found_indexes_but_should_not.len() == 1 { + return Err(Error::SwapIndexFoundDuringRename( + found_indexes_but_should_not.into_iter().next().unwrap().clone(), + )); + } else { + return Err(Error::SwapIndexesFoundDuringRename( + found_indexes_but_should_not.into_iter().cloned().collect(), + )); + } + } progress.update_progress(SwappingTheIndexes::SwappingTheIndexes); for (step, swap) in swaps.iter().enumerate() { progress.update_progress(VariableNameStep::::new( @@ -392,6 +408,7 @@ impl IndexScheduler { task.uid, &swap.indexes.0, &swap.indexes.1, + swap.rename, )?; } wtxn.commit()?; @@ -481,6 +498,7 @@ impl IndexScheduler { task_id: u32, lhs: &str, rhs: &str, + rename: bool, ) -> Result<()> { progress.update_progress(InnerSwappingTwoIndexes::RetrieveTheTasks); // 1. Verify that both lhs and rhs are existing indexes @@ -488,16 +506,23 @@ impl IndexScheduler { if !index_lhs_exists { return Err(Error::IndexNotFound(lhs.to_owned())); } - let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; - if !index_rhs_exists { - return Err(Error::IndexNotFound(rhs.to_owned())); + if !rename { + let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; + if !index_rhs_exists { + return Err(Error::IndexNotFound(rhs.to_owned())); + } } // 2. Get the task set for index = name that appeared before the index swap task let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?; index_lhs_task_ids.remove_range(task_id..); - let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; - index_rhs_task_ids.remove_range(task_id..); + let index_rhs_task_ids = if rename { + let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; + index_rhs_task_ids.remove_range(task_id..); + index_rhs_task_ids + } else { + RoaringBitmap::new() + }; // 3. before_name -> new_name in the task's KindWithContent progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks); @@ -526,7 +551,11 @@ impl IndexScheduler { })?; // 6. Swap in the index mapper - self.index_mapper.swap(wtxn, lhs, rhs)?; + if rename { + self.index_mapper.rename(wtxn, lhs, rhs)?; + } else { + self.index_mapper.swap(wtxn, lhs, rhs)?; + } Ok(()) } diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index e9f21dfe4..8cc1b8830 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -372,8 +372,8 @@ fn swap_indexes() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "d".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "d".to_owned()), rename: false }, ], }, None, @@ -384,7 +384,7 @@ fn swap_indexes() { index_scheduler .register( KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()), rename: false }], }, None, false, @@ -428,8 +428,8 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("b".to_owned(), "a".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("b".to_owned(), "a".to_owned()), rename: false }, ], }, None, @@ -446,9 +446,9 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "e".to_owned()) }, - IndexSwap { indexes: ("d".to_owned(), "f".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "e".to_owned()), rename: false }, + IndexSwap { indexes: ("d".to_owned(), "f".to_owned()), rename: false }, ], }, None, diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 91bba35d7..159e8f3d3 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -271,7 +271,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { } } K::IndexSwap { swaps } => { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -288,7 +288,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { | K::SnapshotCreation => (), }; if let Some(Details::IndexSwap { swaps }) = &mut task.details { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -330,7 +330,7 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { if let KindWithContent::IndexSwap { swaps } = &task.kind { swaps } else { return Ok(()) }; let mut all_indexes = HashSet::new(); let mut duplicate_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps { for name in [lhs, rhs] { let is_new = all_indexes.insert(name); if !is_new { diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 4360d947b..415bb5fdb 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -335,6 +335,7 @@ InvalidState , Internal , INTERNAL InvalidStoreFile , Internal , INTERNAL_SERVER_ERROR ; InvalidSwapDuplicateIndexFound , InvalidRequest , BAD_REQUEST ; InvalidSwapIndexes , InvalidRequest , BAD_REQUEST ; +InvalidSwapRename , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterEnqueuedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterFinishedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterStartedAt , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index f7b59b299..fbdaac9ce 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -173,6 +173,7 @@ pub enum KindWithContent { #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), + pub rename: bool, } #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] diff --git a/crates/meilisearch/src/routes/swap_indexes.rs b/crates/meilisearch/src/routes/swap_indexes.rs index 4a35d1a6d..533d2caf1 100644 --- a/crates/meilisearch/src/routes/swap_indexes.rs +++ b/crates/meilisearch/src/routes/swap_indexes.rs @@ -4,7 +4,7 @@ use deserr::actix_web::AwebJson; use deserr::Deserr; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::InvalidSwapIndexes; +use meilisearch_types::error::deserr_codes::{InvalidSwapIndexes, InvalidSwapRename}; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; @@ -33,11 +33,15 @@ pub struct SwapIndexesPayload { /// Array of the two indexUids to be swapped #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_swap_indexes)] indexes: Vec, + /// If set to true, instead of swapping the left and right indexes it'll change the name of the first index to the second + #[deserr(default, error = DeserrJsonError)] + rename: bool, } #[derive(Serialize)] struct IndexSwappedAnalytics { swap_operation_number: usize, + rename_used: bool, } impl Aggregate for IndexSwappedAnalytics { @@ -48,6 +52,7 @@ impl Aggregate for IndexSwappedAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(Self { swap_operation_number: self.swap_operation_number.max(new.swap_operation_number), + rename_used: self.rename_used | new.rename_used, }) } @@ -95,11 +100,17 @@ pub async fn swap_indexes( analytics: web::Data, ) -> Result { let params = params.into_inner(); - analytics.publish(IndexSwappedAnalytics { swap_operation_number: params.len() }, &req); + analytics.publish( + IndexSwappedAnalytics { + swap_operation_number: params.len(), + rename_used: params.iter().any(|obj| obj.rename), + }, + &req, + ); let filters = index_scheduler.filters(); let mut swaps = vec![]; - for SwapIndexesPayload { indexes } in params.into_iter() { + for SwapIndexesPayload { indexes, rename } in params.into_iter() { // TODO: switch to deserr let (lhs, rhs) = match indexes.as_slice() { [lhs, rhs] => (lhs, rhs), @@ -110,7 +121,7 @@ pub async fn swap_indexes( if !filters.is_index_authorized(lhs) || !filters.is_index_authorized(rhs) { return Err(AuthenticationError::InvalidToken.into()); } - swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()) }); + swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()), rename }); } let task = KindWithContent::IndexSwap { swaps }; diff --git a/crates/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs index d136dfc0f..63dfdfe8a 100644 --- a/crates/meilisearch/tests/swap_indexes/errors.rs +++ b/crates/meilisearch/tests/swap_indexes/errors.rs @@ -92,3 +92,20 @@ async fn swap_indexes_bad_indexes() { } "###); } + +#[actix_rt::test] +async fn swap_indexes_bad_rename() { + let server = Server::new_shared(); + + let (response, code) = + server.index_swap(json!([{ "indexes": ["kefir", "intel"], "rename": "hello" }])).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r#" + { + "message": "Invalid value type at `[0].rename`: expected a boolean, but found a string: `\"hello\"`", + "code": "invalid_swap_rename", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_swap_rename" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs index bf84d5823..780dc2d86 100644 --- a/crates/meilisearch/tests/swap_indexes/mod.rs +++ b/crates/meilisearch/tests/swap_indexes/mod.rs @@ -372,3 +372,255 @@ async fn swap_indexes() { let (res, _) = d.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(res["results"], @r###"[{"id":1,"index":"c"}]"###); } + +#[actix_rt::test] +async fn swap_rename_indexes() { + let server = Server::new().await; + let a = server.index("a"); + let b = server.index("b"); + a.create(None).await; + a.add_documents(json!({ "id": 1, "index": "a"}), None).await; + + let (res, _code) = server.index_swap(json!([{ "indexes": ["a", "b"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + let (tasks, _code) = server.tasks().await; + + // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 2, + "batchUid": 2, + "indexUid": null, + "status": "succeeded", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "a", + "b" + ], + "rename": true + } + ] + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 2, + "next": null + } + "#); + + // BUT, `a` should not exists + let (res, code) = a.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"404 Not Found"); + snapshot!(res["results"], @"null"); + + // And its data should be in b + let (res, code) = b.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"200 OK"); + snapshot!(res["results"], @r#"[{"id":1,"index":"a"}]"#); + + // No tasks should be linked to the index a + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 1, + "limit": 20, + "from": null, + "next": null + } + "#); + + // They should be linked to the index b + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 1, + "next": null + } + "#); + + // ===== Now, we can delete the index `b`, but its tasks will stays + // if we then make a new `b` index and rename it to be called `a` + // the tasks currently available in `b` should not be moved + + let (res, _) = b.delete().await; + server.wait_task(res.uid()).await.succeeded(); + + let (res, _) = b.create(Some("kefir")).await; + let (res, _code) = server.index_swap(json!([{ "indexes": ["b", "a"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + // `a` now contains everything + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 4, + "batchUid": 4, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": "kefir" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 3, + "batchUid": 3, + "indexUid": "a", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "a", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 6, + "limit": 20, + "from": 4, + "next": null + } + "#); + + // And `b` is empty + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 2, + "limit": 20, + "from": null, + "next": null + } + "#); +} From a904ce109af0526dd060d098fc1aa4d1254ddcac Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 17:37:30 +0200 Subject: [PATCH 65/76] fix error code and add a bunch of tests for the swap and index rename --- crates/index-scheduler/src/error.rs | 12 +-- .../src/scheduler/process_batch.rs | 16 ++- crates/meilisearch/tests/batches/mod.rs | 7 +- .../meilisearch/tests/index/update_index.rs | 47 +++++++-- .../meilisearch/tests/swap_indexes/errors.rs | 97 ++++++++++++++++++- crates/meilisearch/tests/swap_indexes/mod.rs | 28 +++--- crates/meilisearch/tests/tasks/mod.rs | 14 +-- 7 files changed, 182 insertions(+), 39 deletions(-) diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 742576c73..332b7e040 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -67,8 +67,8 @@ pub enum Error { SwapDuplicateIndexesFound(Vec), #[error("Index `{0}` not found.")] SwapIndexNotFound(String), - #[error("Index `{0}` found during a rename. Renaming doen't overwrite the other index name.")] - SwapIndexFoundDuringRename(String), + #[error("Cannot rename `{0}` to `{1}` as the index already exists. Hint: You can remove `{1}` first and then do your remove.")] + SwapIndexFoundDuringRename(String, String), #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")] NoSpaceLeftInTaskQueue, #[error( @@ -76,7 +76,7 @@ pub enum Error { .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesNotFound(Vec), - #[error("Index {} found during a rename. Renaming doen't overwrite the other index name.", + #[error("The following indexes are being renamed but cannot because their new name conflicts with an already existing index: {}. Renaming doesn't overwrite the other index name.", .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesFoundDuringRename(Vec), @@ -209,7 +209,7 @@ impl Error { | Error::SwapIndexNotFound(_) | Error::NoSpaceLeftInTaskQueue | Error::SwapIndexesNotFound(_) - | Error::SwapIndexFoundDuringRename(_) + | Error::SwapIndexFoundDuringRename(_, _) | Error::SwapIndexesFoundDuringRename(_) | Error::CorruptedDump | Error::InvalidTaskDate { .. } @@ -279,8 +279,8 @@ impl ErrorCode for Error { Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound, Error::SwapIndexNotFound(_) => Code::IndexNotFound, Error::SwapIndexesNotFound(_) => Code::IndexNotFound, - Error::SwapIndexFoundDuringRename(_) => Code::IndexNotFound, - Error::SwapIndexesFoundDuringRename(_) => Code::IndexNotFound, + Error::SwapIndexFoundDuringRename(_, _) => Code::IndexAlreadyExists, + Error::SwapIndexesFoundDuringRename(_) => Code::IndexAlreadyExists, Error::InvalidTaskDate { field, .. } => (*field).into(), Error::InvalidTaskUid { .. } => Code::InvalidTaskUids, Error::InvalidBatchUid { .. } => Code::InvalidBatchUids, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 7688b1787..738df7b5e 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -368,7 +368,7 @@ impl IndexScheduler { } let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?; match (index_exists, rename) { - (true, true) => found_indexes_but_should_not.insert(rhs), + (true, true) => found_indexes_but_should_not.insert((lhs, rhs)), (false, false) => not_found_indexes.insert(rhs), (true, false) | (false, true) => true, // random value we don't read it anyway }; @@ -386,12 +386,18 @@ impl IndexScheduler { } if !found_indexes_but_should_not.is_empty() { if found_indexes_but_should_not.len() == 1 { - return Err(Error::SwapIndexFoundDuringRename( - found_indexes_but_should_not.into_iter().next().unwrap().clone(), - )); + let (lhs, rhs) = found_indexes_but_should_not + .into_iter() + .next() + .map(|(lhs, rhs)| (lhs.clone(), rhs.clone())) + .unwrap(); + return Err(Error::SwapIndexFoundDuringRename(lhs, rhs)); } else { return Err(Error::SwapIndexesFoundDuringRename( - found_indexes_but_should_not.into_iter().cloned().collect(), + found_indexes_but_should_not + .into_iter() + .map(|(_, rhs)| rhs.to_string()) + .collect(), )); } } diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 9d6bee7c1..4477d93e8 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1142,7 +1142,7 @@ async fn test_summarized_index_swap() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, - @r###" + @r#" { "uid": "[uid]", "progress": null, @@ -1152,7 +1152,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -1172,7 +1173,7 @@ async fn test_summarized_index_swap() { "finishedAt": "[date]", "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." } - "###); + "#); let doggos_index = server.unique_index(); doggos_index.create(None).await; diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 45f3ea420..8e5837d81 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -4,7 +4,9 @@ use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; use crate::common::encoder::Encoder; -use crate::common::{shared_does_not_exists_index, shared_index_with_documents, Server}; +use crate::common::{ + shared_does_not_exists_index, shared_empty_index, shared_index_with_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -113,17 +115,14 @@ async fn error_update_unexisting_index() { async fn update_index_name() { let server = Server::new_shared(); let index = server.unique_index(); - let (task, code) = index.create(None).await; - - assert_eq!(code, 202); + let (task, _code) = index.create(None).await; server.wait_task(task.uid()).await.succeeded(); let new_index = server.unique_index(); - let (task, _status_code) = index.update_raw(json!({ "uid": new_index.uid })).await; + let (task, _code) = index.update_raw(json!({ "uid": new_index.uid })).await; server.wait_task(task.uid()).await.succeeded(); let (response, code) = new_index.get().await; - snapshot!(code, @"200 OK"); assert_eq!(response["uid"], new_index.uid); @@ -139,3 +138,39 @@ async fn update_index_name() { snapshot!(response["primaryKey"], @"null"); snapshot!(response.as_object().unwrap().len(), @"4"); } + +#[actix_rt::test] +async fn error_update_index_name_to_already_existing_index() { + let server = Server::new_shared(); + let base_index = shared_empty_index().await; + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + + let (task, _status_code) = index.update_raw(json!({ "uid": base_index.uid })).await; + let task = server.wait_task(task.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "failed", + "type": "indexUpdate", + "canceledBy": null, + "details": { + "primaryKey": null, + "newIndexUid": "EMPTY_INDEX" + }, + "error": { + "message": "Index `EMPTY_INDEX` already exists.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs index 63dfdfe8a..ee992372e 100644 --- a/crates/meilisearch/tests/swap_indexes/errors.rs +++ b/crates/meilisearch/tests/swap_indexes/errors.rs @@ -1,6 +1,9 @@ use meili_snap::*; -use crate::common::Server; +use crate::common::{ + shared_empty_index, shared_index_with_documents, shared_index_with_geo_documents, + shared_index_with_nested_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -109,3 +112,95 @@ async fn swap_indexes_bad_rename() { } "#); } + +#[actix_rt::test] +async fn swap_indexes_rename_to_already_existing_index() { + let server = Server::new_shared(); + let already_existing_index = shared_empty_index().await; + let base_index = shared_index_with_documents().await; + + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + } + ] + }, + "error": { + "message": "Cannot rename `SHARED_DOCUMENTS` to `EMPTY_INDEX` as the index already exists. Hint: You can remove `EMPTY_INDEX` first and then do your remove.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let base_index_2 = shared_index_with_geo_documents().await; + let already_existing_index_2 = shared_index_with_nested_documents().await; + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }, { "indexes": [base_index_2.uid, already_existing_index_2.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + }, + { + "indexes": [ + "SHARED_GEO_DOCUMENTS", + "SHARED_NESTED_DOCUMENTS" + ], + "rename": true + } + ] + }, + "error": { + "message": "The following indexes are being renamed but cannot because their new name conflicts with an already existing index: `EMPTY_INDEX`, `SHARED_NESTED_DOCUMENTS`. Renaming doesn't overwrite the other index name.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs index 780dc2d86..edb7fab49 100644 --- a/crates/meilisearch/tests/swap_indexes/mod.rs +++ b/crates/meilisearch/tests/swap_indexes/mod.rs @@ -73,7 +73,7 @@ async fn swap_indexes() { snapshot!(code, @"200 OK"); // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -89,7 +89,8 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false } ] }, @@ -102,7 +103,7 @@ async fn swap_indexes() { { "uid": 1, "batchUid": 1, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -139,7 +140,7 @@ async fn swap_indexes() { "from": 2, "next": null } - "###); + "#); // BUT, the data in `a` should now points to the data that was in `b`. // And the opposite is true as well @@ -228,7 +229,7 @@ async fn swap_indexes() { // 2. stays unchanged // 3. now have the indexUid `d` instead of `c` // 4. now have the indexUid `c` instead of `d` - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -244,13 +245,15 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false }, { "indexes": [ "c", "d" - ] + ], + "rename": false } ] }, @@ -263,7 +266,7 @@ async fn swap_indexes() { { "uid": 4, "batchUid": 4, - "indexUid": "c", + "indexUid": "d", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -307,7 +310,8 @@ async fn swap_indexes() { "indexes": [ "b", "a" - ] + ], + "rename": false } ] }, @@ -337,7 +341,7 @@ async fn swap_indexes() { { "uid": 0, "batchUid": 0, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -357,7 +361,7 @@ async fn swap_indexes() { "from": 5, "next": null } - "###); + "#); // - The data in `a` should point to `a`. // - The data in `b` should point to `b`. @@ -530,7 +534,7 @@ async fn swap_rename_indexes() { let (res, _) = b.delete().await; server.wait_task(res.uid()).await.succeeded(); - let (res, _) = b.create(Some("kefir")).await; + b.create(Some("kefir")).await; let (res, _code) = server.index_swap(json!([{ "indexes": ["b", "a"], "rename": true }])).await; server.wait_task(res.uid()).await.succeeded(); diff --git a/crates/meilisearch/tests/tasks/mod.rs b/crates/meilisearch/tests/tasks/mod.rs index 09700d3c5..6397ad6ad 100644 --- a/crates/meilisearch/tests/tasks/mod.rs +++ b/crates/meilisearch/tests/tasks/mod.rs @@ -895,7 +895,7 @@ async fn test_summarized_index_swap() { server.wait_task(task.uid()).await.failed(); let (task, _) = server.get_task(task.uid()).await; snapshot!(task, - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -909,7 +909,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -924,7 +925,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); let doggos_index = server.unique_index(); let (task, _code) = doggos_index.create(None).await; @@ -941,7 +942,7 @@ async fn test_summarized_index_swap() { let (task, _) = server.get_task(task.uid()).await; snapshot!(json_string!(task, { ".uid" => "[uid]", ".batchUid" => "[batch_uid]", ".**.indexes[0]" => "doggos", ".**.indexes[1]" => "cattos", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }), - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -955,7 +956,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -965,7 +967,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); } #[actix_web::test] From 4068c58417bf72aafb9bbdb318f47fd0882fe67e Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 18:28:00 +0200 Subject: [PATCH 66/76] change the details of the tasks --- crates/dump/src/lib.rs | 6 ++-- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +-- crates/index-scheduler/src/dump.rs | 4 +-- crates/index-scheduler/src/insta_snapshot.rs | 4 +-- .../query_batches_canceled_by/start.snap | 8 ++--- .../processed_all_tasks.snap | 6 ++-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 7 ++-- .../after-advancing-a-bit.snap | 6 ++-- .../query_batches_simple/end.snap | 6 ++-- .../query_batches_simple/start.snap | 7 ++-- .../after-processing-everything.snap | 12 +++---- .../query_batches_special_rules/start.snap | 9 +++-- .../query_tasks_canceled_by/start.snap | 8 ++--- .../processed_all_tasks.snap | 6 ++-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 7 ++-- .../tasks_test.rs/query_tasks_simple/end.snap | 6 ++-- .../query_tasks_simple/start.snap | 7 ++-- .../query_tasks_special_rules/start.snap | 9 +++-- ...everything_is_successfully_registered.snap | 3 +- .../after_the_second_task_deletion.snap | 4 +-- .../task_deletion_have_been_enqueued.snap | 13 ++++--- .../task_deletion_have_been_processed.snap | 7 ++-- .../task_queue_is_full.snap | 13 ++++--- .../task_deletion_have_not_been_enqueued.snap | 13 ++++--- .../task_queue_is_full.snap | 13 ++++--- .../src/scheduler/process_batch.rs | 28 +++++++-------- .../all_tasks_processed.snap | 6 ++-- .../before_index_creation.snap | 2 +- .../both_task_succeeded.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 3 +- .../registered_the_third_task.snap | 3 +- .../after_batch_creation.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 4 +-- .../registered_the_third_task.snap | 4 +-- .../processed_the_first_task.snap | 4 +-- .../processed_the_second_task.snap | 4 +-- .../processed_the_third_task.snap | 4 +-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 5 ++- .../first.snap | 2 +- .../fourth.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_fourth_task.snap | 3 +- .../registered_the_second_task.snap | 3 +- .../registered_the_third_task.snap | 3 +- .../second.snap | 2 +- .../third.snap | 2 +- .../test.rs/swap_indexes/create_a.snap | 8 ++--- .../test.rs/swap_indexes/create_b.snap | 8 ++--- .../test.rs/swap_indexes/create_c.snap | 8 ++--- .../test.rs/swap_indexes/create_d.snap | 8 ++--- .../swap_indexes/first_swap_processed.snap | 22 ++++++------ .../swap_indexes/first_swap_registered.snap | 10 +++--- .../swap_indexes/second_swap_processed.snap | 24 ++++++------- .../third_empty_swap_processed.snap | 24 ++++++------- .../swap_indexes/two_swaps_registered.snap | 12 +++---- .../after_the_index_creation.snap | 8 ++--- .../first_swap_failed.snap | 12 +++---- .../initial_tasks_processed.snap | 8 ++--- .../initial_tasks_enqueued.snap | 3 +- .../task_deletion_done.snap | 2 +- .../task_deletion_enqueued.snap | 3 +- .../task_deletion_processing.snap | 2 +- .../after_restart.snap | 2 +- .../registered_task.snap | 2 +- .../registered_a_task.snap | 3 +- .../after_processing_the_10_tasks.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../five_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_register.snap | 3 +- .../index_creation_failed.snap | 2 +- .../index_creation_failed.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_processing_everything.snap | 8 ++--- .../after_removing_the_upgrade_tasks.snap | 8 ++--- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 2 +- .../upgrade_task_failed_again.snap | 4 +-- .../upgrade_task_succeeded.snap | 6 ++-- crates/meilisearch-types/src/error.rs | 1 - crates/meilisearch-types/src/task_view.rs | 4 +-- crates/meilisearch-types/src/tasks.rs | 14 ++++---- crates/meilisearch/src/routes/indexes/mod.rs | 1 - crates/meilisearch/tests/index/errors.rs | 34 +++++-------------- .../meilisearch/tests/index/update_index.rs | 19 +++++++++++ 102 files changed, 300 insertions(+), 320 deletions(-) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index b4b339f09..f56729ce5 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -129,7 +129,7 @@ pub enum KindDump { }, IndexUpdate { primary_key: Option, - new_uid: Option, + uid: Option, }, IndexSwap { swaps: Vec, @@ -211,8 +211,8 @@ impl From for KindDump { KindWithContent::IndexCreation { primary_key, .. } => { KindDump::IndexCreation { primary_key } } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { - KindDump::IndexUpdate { primary_key, new_uid: new_index_uid } + KindWithContent::IndexUpdate { primary_key, new_index_uid: uid, .. } => { + KindDump::IndexUpdate { primary_key, uid } } KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps }, KindWithContent::TaskCancelation { query, tasks } => { diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 790c239d7..9415fa234 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -85,7 +85,7 @@ impl CompatV5ToV6 { v6::Kind::IndexCreation { primary_key } } v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => { - v6::Kind::IndexUpdate { primary_key, new_uid: None } + v6::Kind::IndexUpdate { primary_key, uid: None } } v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion, v5::tasks::TaskContent::DocumentAddition { @@ -141,7 +141,7 @@ impl CompatV5ToV6 { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key, new_uid: None } + v6::Details::IndexInfo { primary_key, uid: None } } v5::Details::DocumentDeletion { received_document_ids, diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 18c665ca3..3f56d63e5 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -197,10 +197,10 @@ impl<'a> Dump<'a> { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, }, - KindDump::IndexUpdate { primary_key, new_uid } => KindWithContent::IndexUpdate { + KindDump::IndexUpdate { primary_key, uid } => KindWithContent::IndexUpdate { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, - new_index_uid: new_uid, + new_index_uid: uid, }, KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps }, KindDump::TaskCancelation { query, tasks } => { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 6d72e4b9f..caef2da39 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key, new_uid } => { - format!("{{ primary_key: {primary_key:?}, new_uid: {new_uid:?} }}") + Details::IndexInfo { primary_key, uid } => { + format!("{{ primary_key: {primary_key:?}, new_uid: {uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap index 48d1ccaab..384db97c8 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap index 4c54de49a..4a9c47047 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap index 74c4c4a33..1247d6029 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap index 411e82ea0..c1777be44 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap index 4c76db95e..a13db4b5e 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap index 7ce0d3ca3..188827723 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap index 603544991..1f136875b 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap index 6dc897dfa..349d9b5eb 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap index 84d6c7878..8f6550ab5 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -54,8 +54,8 @@ doggo: { number_of_documents: 0, field_distribution: {} } ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } -2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } -3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } +2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } +3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap index 1184c197f..a22808394 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap index e3c26b2b3..49bd39572 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap index 4475c71fc..e7879f6b6 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap index 4f9ffb209..b41dc9d4b 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap index eb6b0e7ec..6bbec14b3 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap index 181f0308c..af07454db 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap index 4ac6201a6..062c248ad 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap index 268f463aa..dcb989d9f 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap index 60c041c05..bdcd9be60 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap index e4d9af541..8d1aa089c 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap index 30e8e17a8..9cb42f9e8 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap index 4e3fb5439..62af47a08 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap index 4cabce94b..6b59db7e8 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -34,7 +34,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 738df7b5e..7c5469831 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -240,23 +240,19 @@ impl IndexScheduler { // Handle rename if new_index_uid is provided let final_index_uid = if let Some(new_uid) = &new_index_uid { - index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; + if new_uid != &index_uid { + index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; - let mut wtxn = self.env.write_txn()?; + let mut wtxn = self.env.write_txn()?; + self.apply_index_swap( + &mut wtxn, &progress, task.uid, &index_uid, new_uid, true, + )?; + wtxn.commit()?; - // Rename the index - self.index_mapper.rename(&mut wtxn, &index_uid, new_uid)?; - - // Update the task index mappings - let old_tasks = - self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); - self.queue.tasks.update_index(&mut wtxn, new_uid, |bm| { - *bm |= &old_tasks; - })?; - self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; - wtxn.commit()?; - - new_uid.clone() + new_uid.clone() + } else { + new_uid.clone() + } } else { index_uid.clone() }; @@ -287,7 +283,7 @@ impl IndexScheduler { task.status = Status::Succeeded; task.details = Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }); // if the update processed successfully, we're going to store the new diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap index ed6e75a3d..5f8bd0184 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap index f98e5d308..306d11d6c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap index ae1139c0c..c8565036d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap index 92f24508c..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap index 5f25c2964..4e1f511d7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap index 0006ee8c0..7db4eccbd 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap index 17b69061a..784fa24c0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap index cf2b2b691..de152c46e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap index c8a407554..0b9cc0d2c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap index 0cae69a70..d4bea7912 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "index_a" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap index c5e3e66c8..1e62bab03 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap index 8da1b6ca8..0a8e64b3d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap index 8ee0bfcef..677c0e204 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap index 92f24508c..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap index 21a6a59f7..17c0d6c9a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap index adf9a76fe..ba9ee1c27 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap index a7215f32c..42b9fa416 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap index 1c14b091f..20bfa8041 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap index 5c8082f72..62bf45bfe 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap index a22004697..76d7f8a50 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap index 635491dc1..ab090cac3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap index 1d190baca..f0c0c7652 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap index da91440ab..1ec28ed04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap index 95bc1f7f5..b2607fcfa 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap index 4878bbe28..78eb9cb45 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap index 5a851f373..9975088a5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap index dad7609d2..bdd6e8c14 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap index 39d1b3339..97015c2c6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [5,] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [1,4,5,] -b [0,4,] -c [3,4,5,] -d [2,4,] +a [4,5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -64,7 +64,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap index 5d292fe21..01ea109b0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap index 9327015c4..92a67e022 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -66,8 +66,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap index f85735397..2da41da39 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }} ---------------------------------------------------------------------- ### Status: @@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,] "indexSwap" [4,5,6,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -70,8 +70,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } 6 {uid: 6, details: {"swaps":[]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 6 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap index 46d70dceb..0ce2d02ef 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,5,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap index da9340f3b..cbb84dc48 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -65,7 +65,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","e"]},{"indexes":["d","f"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","e"],"rename":false},{"indexes":["d","f"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index 46cbaefc2..bc4b0661e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap index 3b89fe1e7..8961e5280 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 0, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index a861fea12..761a8eedb 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap index d8abc1314..b2492956b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"matchedTasks":2,"deletedTasks":null,"originalFilter":"test_query"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"taskDeletion":1},"indexUids":{}}, stop reason: "stopped after the last task of type `taskDeletion` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap index 1dde1a394..ad1580038 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap index dd1d76f55..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap index cf2b2b691..de152c46e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap index f8caaa995..bff14da44 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap index d987d66c0..58e75739d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap index d1369460f..a274275a6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap index 03d4e5b16..788c48d04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap index 136777fcf..96d175a3a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap index 0b4fc96b5..9510f7ee5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap index d938ca288..600602315 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap index 2d936ba68..39da9349d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap index 6f0f9c782..c982042d4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap index 6add8a2a5..7e7f5e0b2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap index 197ed0679..581dd82a3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap index d1369460f..a274275a6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap index 03d4e5b16..788c48d04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap index 4ece15b13..302ef6200 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index 8feeaf990..4c7ed728c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap index 201680d7a..5bf7186e2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap index 4ece15b13..302ef6200 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index d700dd3db..dcb1d96fc 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -7,10 +7,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index fb682053c..45aed3211 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index abaffbb1b..3fcf91f9f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 9569ecfe3..8bd521e8c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 1d7945023..9e7073b0c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 869d1d0b2..48398a9cb 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 415bb5fdb..ab924c9f7 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -212,7 +212,6 @@ ImmutableApiKeyKey , InvalidRequest , BAD_REQU ImmutableApiKeyUid , InvalidRequest , BAD_REQUEST; ImmutableApiKeyUpdatedAt , InvalidRequest , BAD_REQUEST; ImmutableIndexCreatedAt , InvalidRequest , BAD_REQUEST; -ImmutableIndexUid , InvalidRequest , BAD_REQUEST; ImmutableIndexUpdatedAt , InvalidRequest , BAD_REQUEST; IndexAlreadyExists , InvalidRequest , CONFLICT ; IndexCreationFailed , Internal , INTERNAL_SERVER_ERROR; diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 460ae68d7..dc1106766 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -341,9 +341,9 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key, new_uid } => DetailsView { + Details::IndexInfo { primary_key, uid } => DetailsView { primary_key: Some(primary_key), - new_index_uid: new_uid.clone(), + new_index_uid: uid.clone(), ..DetailsView::default() }, Details::DocumentDeletion { diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index fbdaac9ce..cdd7f5fed 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -279,12 +279,12 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { swaps } => { @@ -358,12 +358,12 @@ impl KindWithContent { } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => { @@ -419,12 +419,12 @@ impl From<&KindWithContent> for Option
{ } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => None, @@ -678,7 +678,7 @@ pub enum Details { }, IndexInfo { primary_key: Option, - new_uid: Option, + uid: Option, }, DocumentDeletion { provided_ids: usize, diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 632922542..d51ce6c6c 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -288,7 +288,6 @@ fn deny_immutable_fields_index( location: ValuePointerRef, ) -> DeserrJsonError { match field { - "uid" => immutable_field_error(field, accepted, Code::ImmutableIndexUid), "createdAt" => immutable_field_error(field, accepted, Code::ImmutableIndexCreatedAt), "updatedAt" => immutable_field_error(field, accepted, Code::ImmutableIndexUpdatedAt), _ => deserr::take_cf_content(DeserrJsonError::::error::( diff --git a/crates/meilisearch/tests/index/errors.rs b/crates/meilisearch/tests/index/errors.rs index 3bab83955..2f10a5b67 100644 --- a/crates/meilisearch/tests/index/errors.rs +++ b/crates/meilisearch/tests/index/errors.rs @@ -160,36 +160,20 @@ async fn update_index_bad_primary_key() { "###); } -#[actix_rt::test] -async fn update_index_immutable_uid() { - let server = Server::new_shared(); - let index = server.unique_index(); - let (response, code) = index.update_raw(json!({ "uid": "doggo" })).await; - snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" - { - "message": "Immutable field `uid`: expected one of `primaryKey`", - "code": "immutable_index_uid", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#immutable_index_uid" - } - "###); -} - #[actix_rt::test] async fn update_index_immutable_created_at() { let server = Server::new_shared(); let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "createdAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `createdAt`: expected one of `primaryKey`", + "message": "Immutable field `createdAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_created_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_created_at" } - "###); + "#); } #[actix_rt::test] @@ -198,14 +182,14 @@ async fn update_index_immutable_updated_at() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "updatedAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `updatedAt`: expected one of `primaryKey`", + "message": "Immutable field `updatedAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_updated_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_updated_at" } - "###); + "#); } #[actix_rt::test] @@ -214,14 +198,14 @@ async fn update_index_unknown_field() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "doggo": "bork" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Unknown field `doggo`: expected one of `primaryKey`", + "message": "Unknown field `doggo`: expected one of `primaryKey`, `uid`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 8e5837d81..97b31ec68 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -139,6 +139,25 @@ async fn update_index_name() { snapshot!(response.as_object().unwrap().len(), @"4"); } +#[actix_rt::test] +async fn update_index_name_to_itself() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + let (initial_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + let (task, _code) = index.update_raw(json!({ "uid": index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (new_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + // Renaming an index to its own name should not change anything + assert_eq!(initial_response, new_response); +} + #[actix_rt::test] async fn error_update_index_name_to_already_existing_index() { let server = Server::new_shared(); From 81020c7d6d2bb6365e3b15f57cda25b5815e2bd8 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 18:38:21 +0200 Subject: [PATCH 67/76] remove a duplicated test --- .../meilisearch/tests/index/rename_index.rs | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/crates/meilisearch/tests/index/rename_index.rs b/crates/meilisearch/tests/index/rename_index.rs index 793b079f1..23109f272 100644 --- a/crates/meilisearch/tests/index/rename_index.rs +++ b/crates/meilisearch/tests/index/rename_index.rs @@ -390,30 +390,3 @@ async fn rename_index_with_pending_tasks() { let docs = response["results"].as_array().unwrap(); assert!(!docs.is_empty()); // At least the initial document should be there } - -#[actix_rt::test] -async fn rename_index_to_same_name() { - let server = Server::new_shared(); - let index = server.unique_index(); - - // Create index - let (task, code) = index.create(None).await; - assert_eq!(code, 202); - server.wait_task(task.uid()).await.succeeded(); - - // Try to rename to the same name - let body = json!({ "uid": index.uid }); - let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; - - assert_eq!(code, 202); - let response = server.wait_task(task.uid()).await.failed(); - - // Should fail with index already exists error - assert_eq!(response["status"], "failed"); - assert_eq!(response["type"], "indexUpdate"); - assert_eq!(response["error"]["code"], "index_already_exists"); - - // Index should still be accessible with original name - let (_, code) = index.get().await; - assert_eq!(code, 200); -} From 2bab375001700a8fa5b14f9d650687932bb3f5b4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 19:33:15 +0200 Subject: [PATCH 68/76] update the task details again --- crates/dump/src/reader/compat/v5_to_v6.rs | 8 ++-- crates/index-scheduler/src/insta_snapshot.rs | 4 +- .../query_batches_canceled_by/start.snap | 4 +- .../processed_all_tasks.snap | 6 +-- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 6 +-- .../after-advancing-a-bit.snap | 6 +-- .../query_batches_simple/end.snap | 6 +-- .../query_batches_simple/start.snap | 6 +-- .../after-processing-everything.snap | 4 +- .../query_batches_special_rules/start.snap | 4 +- .../query_tasks_canceled_by/start.snap | 4 +- .../processed_all_tasks.snap | 6 +-- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 6 +-- .../tasks_test.rs/query_tasks_simple/end.snap | 6 +-- .../query_tasks_simple/start.snap | 6 +-- .../query_tasks_special_rules/start.snap | 4 +- ...everything_is_successfully_registered.snap | 2 +- .../after_the_second_task_deletion.snap | 3 +- .../task_deletion_have_been_enqueued.snap | 12 ++++-- .../task_deletion_have_been_processed.snap | 6 ++- .../task_queue_is_full.snap | 12 ++++-- .../task_deletion_have_not_been_enqueued.snap | 12 ++++-- .../task_queue_is_full.snap | 12 ++++-- .../src/scheduler/process_batch.rs | 4 +- .../all_tasks_processed.snap | 6 +-- .../before_index_creation.snap | 2 +- .../both_task_succeeded.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 2 +- .../registered_the_third_task.snap | 2 +- .../after_batch_creation.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 4 +- .../processed_the_first_task.snap | 4 +- .../processed_the_second_task.snap | 4 +- .../processed_the_third_task.snap | 4 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 4 +- .../first.snap | 2 +- .../fourth.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_fourth_task.snap | 2 +- .../registered_the_second_task.snap | 2 +- .../registered_the_third_task.snap | 2 +- .../second.snap | 2 +- .../third.snap | 2 +- .../test.rs/swap_indexes/create_a.snap | 8 ++-- .../test.rs/swap_indexes/create_b.snap | 8 ++-- .../test.rs/swap_indexes/create_c.snap | 8 ++-- .../test.rs/swap_indexes/create_d.snap | 8 ++-- .../swap_indexes/first_swap_processed.snap | 8 ++-- .../swap_indexes/first_swap_registered.snap | 8 ++-- .../swap_indexes/second_swap_processed.snap | 8 ++-- .../third_empty_swap_processed.snap | 8 ++-- .../swap_indexes/two_swaps_registered.snap | 8 ++-- .../after_the_index_creation.snap | 8 ++-- .../first_swap_failed.snap | 8 ++-- .../initial_tasks_processed.snap | 8 ++-- .../initial_tasks_enqueued.snap | 2 +- .../task_deletion_done.snap | 2 +- .../task_deletion_enqueued.snap | 2 +- .../task_deletion_processing.snap | 2 +- .../after_restart.snap | 2 +- .../registered_task.snap | 2 +- .../registered_a_task.snap | 2 +- .../after_processing_the_10_tasks.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../five_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_register.snap | 2 +- .../index_creation_failed.snap | 2 +- .../index_creation_failed.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_processing_everything.snap | 8 ++-- .../after_removing_the_upgrade_tasks.snap | 8 ++-- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 2 +- .../upgrade_task_failed_again.snap | 4 +- .../upgrade_task_succeeded.snap | 6 +-- crates/meilisearch-types/src/task_view.rs | 5 ++- crates/meilisearch-types/src/tasks.rs | 42 ++++++++++++------- crates/meilisearch/tests/common/index.rs | 23 ++++++++++ .../meilisearch/tests/index/update_index.rs | 11 +++-- 98 files changed, 269 insertions(+), 213 deletions(-) diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 9415fa234..634604639 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -140,9 +140,11 @@ impl CompatV5ToV6 { v5::Details::Settings { settings } => { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } - v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key, uid: None } - } + v5::Details::IndexInfo { primary_key } => v6::Details::IndexInfo { + primary_key, + new_index_uid: None, + old_index_uid: None, + }, v5::Details::DocumentDeletion { received_document_ids, deleted_documents, diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index caef2da39..e7a068899 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key, uid } => { - format!("{{ primary_key: {primary_key:?}, new_uid: {uid:?} }}") + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => { + format!("{{ primary_key: {primary_key:?}, old_new_uid: {old_index_uid:?}, new_index_uid: {new_index_uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap index 384db97c8..ec05be8f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap index 4a9c47047..a30869296 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap index 1247d6029..9a335acf5 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap index c1777be44..022243e3b 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap index a13db4b5e..08aa4856e 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap index 188827723..fcd523744 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap index 1f136875b..b3443a0ad 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap index 349d9b5eb..2039314cf 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap index 8f6550ab5..1e9eaa775 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap index a22808394..26271c168 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap index 49bd39572..c10efb84a 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap index e7879f6b6..af00b64d0 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap index b41dc9d4b..b56d82bcd 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap index 6bbec14b3..d5e66a7de 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap index af07454db..3e93b86b9 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap index 062c248ad..939f8e9f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap index dcb989d9f..80b1d71be 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap index bdcd9be60..3949d7c90 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap index 8d1aa089c..ae4ff5b21 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap index 9cb42f9e8..51dee86b1 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap index 62af47a08..96d6c0c9a 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap index 6b59db7e8..99e57fec2 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -35,7 +36,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 7c5469831..4129c57af 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -283,7 +283,9 @@ impl IndexScheduler { task.status = Status::Succeeded; task.details = Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + new_index_uid: new_index_uid.clone(), + // we only display the old index uid if a rename happened => there is a new_index_uid + old_index_uid: new_index_uid.map(|_| index_uid.clone()), }); // if the update processed successfully, we're going to store the new diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap index 5f8bd0184..e19a94620 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap index 306d11d6c..ae975d843 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap index c8565036d..298ae423f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap index 4e1f511d7..0fe2d8915 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap index 7db4eccbd..50fb4b954 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap index 784fa24c0..7341d77bc 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap index de152c46e..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap index 0b9cc0d2c..aebb02551 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap index d4bea7912..5eb5d37c6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "index_a" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap index 1e62bab03..5aeff9852 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap index 0a8e64b3d..234bdc6a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap index 677c0e204..1b4f9078c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap index 17c0d6c9a..555ac8fed 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap index ba9ee1c27..d259336c0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap index 42b9fa416..0e5f2a994 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap index 20bfa8041..0643dec6b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap index 62bf45bfe..9604b01ad 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap index 76d7f8a50..ae611686b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap index ab090cac3..7b995cf10 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap index f0c0c7652..9ac27f0e7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap index 1ec28ed04..e6f48e5fe 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap index b2607fcfa..8759072a4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap index 78eb9cb45..37afc3447 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap index 9975088a5..683d60cef 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap index bdd6e8c14..d2f999560 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap index 97015c2c6..e7b05d39e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} 5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap index 01ea109b0..ce45dfe50 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap index 92a67e022..0c23e839b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap index 2da41da39..3c9dbc998 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap index 0ce2d02ef..c79a43013 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} 5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap index cbb84dc48..604db5003 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index bc4b0661e..b8979eb88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap index 8961e5280..daaeb92be 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 0, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index 761a8eedb..fe9332a88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap index b2492956b..9ab62e8d7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"matchedTasks":2,"deletedTasks":null,"originalFilter":"test_query"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"taskDeletion":1},"indexUids":{}}, stop reason: "stopped after the last task of type `taskDeletion` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap index ad1580038..c2d6d5f07 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap index de152c46e..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap index bff14da44..8cc9bd96b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap index 58e75739d..08cda75e5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap index a274275a6..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap index 788c48d04..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap index 96d175a3a..6bf7eff90 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap index 9510f7ee5..7c667807c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap index 600602315..26a1524a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap index 39da9349d..a9fcb2536 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap index c982042d4..3bc630256 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap index 7e7f5e0b2..86895e12c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap index 581dd82a3..bb769598c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap index a274275a6..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap index 788c48d04..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap index 302ef6200..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index 4c7ed728c..72c638b4d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap index 5bf7186e2..6b1e7fa32 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap index 302ef6200..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index dcb1d96fc..ed5a60639 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -7,10 +7,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index 45aed3211..54b7e34e1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 3fcf91f9f..bfbecb66e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 8bd521e8c..f29fb35c8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 9e7073b0c..57f395473 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 48398a9cb..3bbd4e51e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index dc1106766..df12d56c6 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -341,9 +341,10 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key, uid } => DetailsView { + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => DetailsView { primary_key: Some(primary_key), - new_index_uid: uid.clone(), + new_index_uid: new_index_uid.clone(), + old_index_uid: old_index_uid.clone(), ..DetailsView::default() }, Details::DocumentDeletion { diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index cdd7f5fed..878a74777 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -278,13 +278,16 @@ impl KindWithContent { KindWithContent::SettingsUpdate { new_settings, .. } => { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { swaps } => { @@ -357,13 +360,16 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => { @@ -418,13 +424,16 @@ impl From<&KindWithContent> for Option
{ Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_index_uid: None, + old_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => None, @@ -678,7 +687,8 @@ pub enum Details { }, IndexInfo { primary_key: Option, - uid: Option, + new_index_uid: Option, + old_index_uid: Option, }, DocumentDeletion { provided_ids: usize, diff --git a/crates/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs index 012c9bebe..f8ff5ced9 100644 --- a/crates/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -319,6 +319,24 @@ impl Index<'_, Shared> { } (task, code) } + + pub async fn update_raw_index_fail( + &self, + body: Value, + waiter: &Server, + ) -> (Value, StatusCode) { + let (mut task, code) = self._update_raw(body).await; + if code.is_success() { + task = waiter.wait_task(task.uid()).await; + if task.is_success() { + panic!( + "`update_raw_index_fail` succeeded: {}", + serde_json::to_string_pretty(&task).unwrap() + ); + } + } + (task, code) + } } #[allow(dead_code)] @@ -370,6 +388,11 @@ impl Index<'_, State> { self.service.patch_encoded(url, body, self.encoder).await } + pub(super) async fn _update_raw(&self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); + self.service.patch_encoded(url, body, self.encoder).await + } + pub(super) async fn _delete(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.delete(url).await diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 97b31ec68..5b72d4328 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -162,22 +162,21 @@ async fn update_index_name_to_itself() { async fn error_update_index_name_to_already_existing_index() { let server = Server::new_shared(); let base_index = shared_empty_index().await; - let index = server.unique_index(); - let (task, _code) = index.create(None).await; - server.wait_task(task.uid()).await.succeeded(); + let index = shared_index_with_documents().await; - let (task, _status_code) = index.update_raw(json!({ "uid": base_index.uid })).await; - let task = server.wait_task(task.uid()).await; + let (task, _status_code) = + index.update_raw_index_fail(json!({ "uid": base_index.uid }), server).await; snapshot!(task, @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "[uuid]", + "indexUid": "SHARED_DOCUMENTS", "status": "failed", "type": "indexUpdate", "canceledBy": null, "details": { "primaryKey": null, + "oldIndexUid": "SHARED_DOCUMENTS", "newIndexUid": "EMPTY_INDEX" }, "error": { From b0479eb996308ddacfbee5c12c83df2d6623fd9a Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 12 Aug 2025 15:44:37 +0200 Subject: [PATCH 69/76] make it work with the dump and dumpless upgrade --- crates/meilisearch-types/src/tasks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 878a74777..eb1f61c58 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -173,6 +173,7 @@ pub enum KindWithContent { #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), + #[serde(default)] pub rename: bool, } From b5ba0e42b32ced20909a961b056a2b329be7c479 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 09:58:16 +0200 Subject: [PATCH 70/76] Add new error --- crates/filter-parser/src/condition.rs | 7 +++++++ crates/filter-parser/src/error.rs | 10 +++++++++- crates/filter-parser/src/lib.rs | 12 ++++++------ crates/filter-parser/src/value.rs | 18 ++++-------------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 4d156c269..24c6c50cc 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -135,6 +135,13 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // From this point, we are certain this is a vector filter, so our errors must be final. // We could use nom's `cut` but it's better to be explicit about the errors + if let Ok((_, space)) = tag::<_, _, ()>(" ")(input) { + return Err(crate::Error::new_failure_from_kind( + space, + ErrorKind::VectorFilterMissingEmbedder, + )); + } + let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index bbf2c8d17..a5905b1cb 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -54,7 +54,7 @@ impl<'a, T> IResultExt<'a> for IResult<'a, T> { nom::Err::Error(e) => *e.context(), nom::Err::Failure(e) => *e.context(), }; - nom::Err::Failure(Error::new_from_kind(input, kind)) + Error::new_failure_from_kind(input, kind) }) } } @@ -79,6 +79,7 @@ pub enum ErrorKind<'a> { MisusedGeoRadius, MisusedGeoBoundingBox, VectorFilterLeftover, + VectorFilterMissingEmbedder, VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, @@ -112,6 +113,10 @@ impl<'a> Error<'a> { Self { context, kind } } + pub fn new_failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { + nom::Err::Failure(Self::new_from_kind(context, kind)) + } + pub fn new_from_external(context: Span<'a>, error: impl std::error::Error) -> Self { Self::new_from_kind(context, ErrorKind::External(error.to_string())) } @@ -199,6 +204,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterMissingFragment => { writeln!(f, "The vector filter is missing a fragment name.")? } + ErrorKind::VectorFilterMissingEmbedder => { + writeln!(f, "Was expecting embedder name but found nothing.")? + } ErrorKind::VectorFilterInvalidEmbedder => { writeln!(f, "The vector filter's embedder is invalid.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index ae11ccf55..8ecbf5dc4 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -437,7 +437,7 @@ fn parse_geo_bounding_box(input: Span) -> IResult { let (input, args) = parsed?; if args.len() != 2 || args[0].len() != 2 || args[1].len() != 2 { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::GeoBoundingBox))); + return Err(Error::new_failure_from_kind(input, ErrorKind::GeoBoundingBox)); } let res = FilterCondition::GeoBoundingBox { @@ -458,7 +458,7 @@ fn parse_geo_point(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))?; // if we succeeded we still return a `Failure` because geoPoints are not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) } /// geoPoint = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float) @@ -472,7 +472,7 @@ fn parse_geo_distance(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?; // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) } /// geo = WS* "_geo(float WS* "," WS* float WS* "," WS* float) @@ -486,7 +486,7 @@ fn parse_geo(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?; // if we succeeded we still return a `Failure` because `_geo` filter is not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) } fn parse_error_reserved_keyword(input: Span) -> IResult { @@ -1006,8 +1006,8 @@ pub mod tests { 1:25 _vectors _vectors EXISTS "); insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" - The vector filter's embedder is invalid. - 10:30 _vectors. embedderName EXISTS + Was expecting embedder name but found nothing. + 10:11 _vectors. embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index 345f0b0a2..ac645799b 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -132,31 +132,21 @@ pub fn parse_value(input: Span) -> IResult { } match parse_geo_radius(input) { - Ok(_) => { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius))) - } + Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)), // if we encountered a failure it means the user badly wrote a _geoRadius filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius))) + return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)) } _ => (), } match parse_geo_bounding_box(input) { - Ok(_) => { - return Err(nom::Err::Failure(Error::new_from_kind( - input, - ErrorKind::MisusedGeoBoundingBox, - ))) - } + Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), // if we encountered a failure it means the user badly wrote a _geoBoundingBox filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(nom::Err::Failure(Error::new_from_kind( - input, - ErrorKind::MisusedGeoBoundingBox, - ))) + return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) } _ => (), } From f6559258ced2f025a8e54dc9be0d7e41be82d0dc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 10:32:28 +0200 Subject: [PATCH 71/76] Improve operation error on vector filters --- crates/filter-parser/src/condition.rs | 27 +++++++++++++++------------ crates/filter-parser/src/error.rs | 4 ++++ crates/filter-parser/src/lib.rs | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 24c6c50cc..12542110a 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -164,21 +164,24 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> Ok((input, (Token::from(fid), Some(embedder_name), filter))) } -/// vectors_exists = vectors "EXISTS" +/// vectors_exists = vectors ("EXISTS" | ("NOT" WS+ "EXISTS")) pub fn parse_vectors_exists(input: Span) -> IResult { - let (input, (fid, embedder, filter)) = terminated(parse_vectors, tag("EXISTS"))(input)?; - - Ok((input, FilterCondition::VectorExists { fid, embedder, filter })) -} -/// vectors_not_exists = vectors "NOT" WS+ "EXISTS" -pub fn parse_vectors_not_exists(input: Span) -> IResult { let (input, (fid, embedder, filter)) = parse_vectors(input)?; - let (input, _) = tuple((tag("NOT"), multispace1, tag("EXISTS")))(input)?; - Ok(( - input, - FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), - )) + // Try parsing "EXISTS" first + if let Ok((input, _)) = tag::<_, _, ()>("EXISTS")(input) { + return Ok((input, FilterCondition::VectorExists { fid, embedder, filter })); + } + + // Try parsing "NOT EXISTS" + if let Ok((input, _)) = tuple::<_, _, (), _>((tag("NOT"), multispace1, tag("EXISTS")))(input) { + return Ok(( + input, + FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), + )); + } + + Err(crate::Error::new_failure_from_kind(input, ErrorKind::VectorFilterOperation)) } /// contains = value "CONTAINS" value diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index a5905b1cb..f6356f3fd 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -83,6 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, + VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, ExpectedEof, @@ -210,6 +211,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterInvalidEmbedder => { writeln!(f, "The vector filter's embedder is invalid.")? } + ErrorKind::VectorFilterOperation => { + writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 8ecbf5dc4..2c8dfac80 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -65,7 +65,7 @@ use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; -use crate::condition::{parse_vectors_exists, parse_vectors_not_exists}; +use crate::condition::parse_vectors_exists; use crate::error::IResultExt; pub type Span<'a> = LocatedSpan<&'a str, &'a str>; @@ -525,7 +525,7 @@ fn parse_primary(input: Span, depth: usize) -> IResult { parse_is_not_null, parse_is_empty, parse_is_not_empty, - alt((parse_vectors_exists, parse_vectors_not_exists, parse_exists, parse_not_exists)), + alt((parse_vectors_exists, parse_exists, parse_not_exists)), parse_to, parse_contains, parse_not_contains, @@ -1002,16 +1002,16 @@ pub mod tests { ); insta::assert_snapshot!(p(r#"_vectors _vectors EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors _vectors EXISTS`. - 1:25 _vectors _vectors EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 10:25 _vectors _vectors EXISTS "); insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" Was expecting embedder name but found nothing. 10:11 _vectors. embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. - 1:30 _vectors .embedderName EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 10:30 _vectors .embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" The vector filter has leftover tokens. @@ -1038,20 +1038,20 @@ pub mod tests { 33:40 _vectors.embedderName.fragments. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName.fragments.test test EXISTS`. - 1:49 _vectors.embedderName.fragments.test test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 38:49 _vectors.embedderName.fragments.test test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" The vector filter's fragment is invalid. 33:45 _vectors.embedderName.fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments. test EXISTS`. - 1:46 _vectors.embedderName .fragments. test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 23:46 _vectors.embedderName .fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments.test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments.test EXISTS`. - 1:45 _vectors.embedderName .fragments.test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 23:45 _vectors.embedderName .fragments.test EXISTS "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" From 666ae1a3e745f167946cacc039ca511bd9fc52c9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:00:38 +0200 Subject: [PATCH 72/76] Add "did you mean" message --- Cargo.lock | 1 + crates/filter-parser/Cargo.toml | 1 + crates/filter-parser/src/condition.rs | 18 ++++++++++++----- crates/filter-parser/src/error.rs | 29 +++++++++++++++++++++++++-- crates/filter-parser/src/lib.rs | 20 ++++++++++-------- crates/filter-parser/src/value.rs | 8 ++++---- 6 files changed, 58 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8413b3d14..43f491f1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2031,6 +2031,7 @@ name = "filter-parser" version = "1.16.0" dependencies = [ "insta", + "levenshtein_automata", "nom", "nom_locate", "unescaper", diff --git a/crates/filter-parser/Cargo.toml b/crates/filter-parser/Cargo.toml index 6eeb0794b..173cabd4b 100644 --- a/crates/filter-parser/Cargo.toml +++ b/crates/filter-parser/Cargo.toml @@ -15,6 +15,7 @@ license.workspace = true nom = "7.1.3" nom_locate = "4.2.0" unescaper = "0.1.6" +levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } [dev-dependencies] # fixed version due to format breakages in v1.40 diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 12542110a..e4795d048 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -19,6 +19,7 @@ use Condition::*; use crate::error::IResultExt; use crate::value::parse_vector_value; +use crate::Error; use crate::ErrorKind; use crate::VectorFilter; use crate::{parse_value, FilterCondition, IResult, Span, Token}; @@ -136,10 +137,7 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // We could use nom's `cut` but it's better to be explicit about the errors if let Ok((_, space)) = tag::<_, _, ()>(" ")(input) { - return Err(crate::Error::new_failure_from_kind( - space, - ErrorKind::VectorFilterMissingEmbedder, - )); + return Err(crate::Error::failure_from_kind(space, ErrorKind::VectorFilterMissingEmbedder)); } let (input, embedder_name) = @@ -159,6 +157,16 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> value(VectorFilter::None, nom::combinator::success("")), ))(input)?; + if let Ok((input, point)) = tag::<_, _, ()>(".")(input) { + let opt_value = parse_vector_value(input).ok().map(|(_, v)| v); + let value = opt_value + .as_ref() + .map(|v| v.original_span().to_string()) + .unwrap_or_else(|| point.to_string()); + let context = opt_value.map(|v| v.original_span()).unwrap_or(point); + return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); + } + let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; Ok((input, (Token::from(fid), Some(embedder_name), filter))) @@ -181,7 +189,7 @@ pub fn parse_vectors_exists(input: Span) -> IResult { )); } - Err(crate::Error::new_failure_from_kind(input, ErrorKind::VectorFilterOperation)) + Err(crate::Error::failure_from_kind(input, ErrorKind::VectorFilterOperation)) } /// contains = value "CONTAINS" value diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index f6356f3fd..91ae2e33c 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -54,7 +54,7 @@ impl<'a, T> IResultExt<'a> for IResult<'a, T> { nom::Err::Error(e) => *e.context(), nom::Err::Failure(e) => *e.context(), }; - Error::new_failure_from_kind(input, kind) + Error::failure_from_kind(input, kind) }) } } @@ -83,6 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, + VectorFilterUnknownSuffix(String), VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, @@ -114,7 +115,7 @@ impl<'a> Error<'a> { Self { context, kind } } - pub fn new_failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { + pub fn failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { nom::Err::Failure(Self::new_from_kind(context, kind)) } @@ -155,6 +156,20 @@ impl Display for Error<'_> { // first line being the diagnostic and the second line being the incriminated filter. let escaped_input = input.escape_debug(); + fn key_suggestion<'a>(key: &str, keys: &[&'a str]) -> Option<&'a str> { + let typos = + levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true).build_dfa(key); + for key in keys.iter() { + match typos.eval(key) { + levenshtein_automata::Distance::Exact(_) => { + return Some(key); + } + levenshtein_automata::Distance::AtLeast(_) => continue, + } + } + None + } + match &self.kind { ErrorKind::ExpectedValue(_) if input.trim().is_empty() => { writeln!(f, "Was expecting a value but instead got nothing.")? @@ -199,6 +214,16 @@ impl Display for Error<'_> { ErrorKind::VectorFilterLeftover => { writeln!(f, "The vector filter has leftover tokens.")? } + ErrorKind::VectorFilterUnknownSuffix(value) if value.as_str() == "." => { + writeln!(f, "Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.")?; + } + ErrorKind::VectorFilterUnknownSuffix(value) => { + if let Some(suggestion) = key_suggestion(value, &["fragments", "userProvided", "documentTemplate", "regenerate"]) { + writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`. Did you mean `{suggestion}`?")?; + } else { + writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`.")?; + } + } ErrorKind::VectorFilterInvalidFragment => { writeln!(f, "The vector filter's fragment is invalid.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 2c8dfac80..75cbecc26 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -437,7 +437,7 @@ fn parse_geo_bounding_box(input: Span) -> IResult { let (input, args) = parsed?; if args.len() != 2 || args[0].len() != 2 || args[1].len() != 2 { - return Err(Error::new_failure_from_kind(input, ErrorKind::GeoBoundingBox)); + return Err(Error::failure_from_kind(input, ErrorKind::GeoBoundingBox)); } let res = FilterCondition::GeoBoundingBox { @@ -458,7 +458,7 @@ fn parse_geo_point(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))?; // if we succeeded we still return a `Failure` because geoPoints are not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) } /// geoPoint = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float) @@ -472,7 +472,7 @@ fn parse_geo_distance(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?; // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) } /// geo = WS* "_geo(float WS* "," WS* float WS* "," WS* float) @@ -486,7 +486,7 @@ fn parse_geo(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?; // if we succeeded we still return a `Failure` because `_geo` filter is not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) } fn parse_error_reserved_keyword(input: Span) -> IResult { @@ -1014,8 +1014,8 @@ pub mod tests { 10:30 _vectors .embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" - The vector filter has leftover tokens. - 22:30 _vectors.embedderName. EXISTS + Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value. + 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" The vector filter's embedder is invalid. @@ -1026,8 +1026,8 @@ pub mod tests { 23:31 _vectors."embedderNam"e EXISTS "#); insta::assert_snapshot!(p(r#"_vectors.embedderName.documentTemplate. EXISTS"#), @r" - The vector filter has leftover tokens. - 39:47 _vectors.embedderName.documentTemplate. EXISTS + Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value. + 39:40 _vectors.embedderName.documentTemplate. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments EXISTS"#), @r" The vector filter is missing a fragment name. @@ -1053,6 +1053,10 @@ pub mod tests { Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. 23:45 _vectors.embedderName .fragments.test EXISTS "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fargments.test EXISTS"#), @r" + Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `fargments`. Did you mean `fragments`? + 23:32 _vectors.embedderName.fargments.test EXISTS + "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index ac645799b..dac96b4f4 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -132,21 +132,21 @@ pub fn parse_value(input: Span) -> IResult { } match parse_geo_radius(input) { - Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)), + Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius)), // if we encountered a failure it means the user badly wrote a _geoRadius filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)) + return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius)) } _ => (), } match parse_geo_bounding_box(input) { - Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), + Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), // if we encountered a failure it means the user badly wrote a _geoBoundingBox filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) + return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) } _ => (), } From b80869f2beaab82705911a10afe8bcfeea2324ab Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:16:25 +0200 Subject: [PATCH 73/76] Add two other "did you mean" messages --- crates/filter-parser/src/condition.rs | 6 ++--- crates/meilisearch/tests/search/filters.rs | 4 +-- crates/milli/src/error.rs | 26 +++++++++++++++++++ .../milli/src/search/facet/filter_vector.rs | 8 +++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index e4795d048..bdc5038e8 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -159,10 +159,8 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> if let Ok((input, point)) = tag::<_, _, ()>(".")(input) { let opt_value = parse_vector_value(input).ok().map(|(_, v)| v); - let value = opt_value - .as_ref() - .map(|v| v.original_span().to_string()) - .unwrap_or_else(|| point.to_string()); + let value = + opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string()); let context = opt_value.map(|v| v.original_span()).unwrap_or(point); return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 8e3ee9249..9cd6575ac 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -952,13 +952,13 @@ async fn vector_filter_non_existant_fragment() { let (value, _code) = index .search_post(json!({ - "filter": "_vectors.rest.fragments.other EXISTS", + "filter": "_vectors.rest.fragments.withBred EXISTS", "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: The fragment `other` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`.\n25:30 _vectors.rest.fragments.other EXISTS", + "message": "Index `[uuid]`: The fragment `withBred` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`. Did you mean `withBreed`?\n25:33 _vectors.rest.fragments.withBred EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 9ad9d0511..76ad3fda0 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -639,3 +639,29 @@ fn conditionally_lookup_for_error_message() { assert_eq!(err.to_string(), format!("{} {}", prefix, suffix)); } } + +pub struct DidYouMean<'a>(Option<&'a str>); + +impl<'a> DidYouMean<'a> { + pub fn new(key: &str, keys: &'a [String]) -> DidYouMean<'a> { + let typos = levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true).build_dfa(key); + for key in keys.iter() { + match typos.eval(key) { + levenshtein_automata::Distance::Exact(_) => { + return DidYouMean(Some(key)); + } + levenshtein_automata::Distance::AtLeast(_) => continue, + } + } + DidYouMean(None) + } +} + +impl std::fmt::Display for DidYouMean<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(suggestion) = self.0 { + write!(f, " Did you mean `{suggestion}`?")?; + } + Ok(()) + } +} diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 625bd5dde..1ef4b8e3d 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,7 +1,7 @@ use filter_parser::{Token, VectorFilter}; use roaring::{MultiOps, RoaringBitmap}; -use crate::error::Error; +use crate::error::{DidYouMean, Error}; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; @@ -14,7 +14,8 @@ pub enum VectorFilterError<'a> { } else { let mut available = available.clone(); available.sort_unstable(); - format!("Available embedders are: {}.", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) + let did_you_mean = DidYouMean::new(embedder.value(), &available); + format!("Available embedders are: {}.{did_you_mean}", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) } })] EmbedderDoesNotExist { embedder: &'a Token<'a>, available: Vec }, @@ -25,7 +26,8 @@ pub enum VectorFilterError<'a> { } else { let mut available = available.clone(); available.sort_unstable(); - format!("Available fragments on this embedder are: {}.", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) + let did_you_mean = DidYouMean::new(fragment.value(), &available); + format!("Available fragments on this embedder are: {}.{did_you_mean}", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) } })] FragmentDoesNotExist { From 8529e2161acc5c3894a1123c320ef02d60b4d380 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:37:19 +0200 Subject: [PATCH 74/76] Clarify more errors --- crates/filter-parser/src/condition.rs | 12 +++++++++++- crates/filter-parser/src/error.rs | 15 +++++++++++---- crates/filter-parser/src/lib.rs | 10 +++++++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index bdc5038e8..c407a1e45 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -162,7 +162,17 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> let value = opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string()); let context = opt_value.map(|v| v.original_span()).unwrap_or(point); - return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); + let previous_kind = match filter { + VectorFilter::Fragment(_) => Some("fragments"), + VectorFilter::DocumentTemplate => Some("documentTemplate"), + VectorFilter::UserProvided => Some("userProvided"), + VectorFilter::Regenerate => Some("regenerate"), + VectorFilter::None => None, + }; + return Err(Error::failure_from_kind( + context, + ErrorKind::VectorFilterUnknownSuffix(previous_kind, value), + )); } let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 91ae2e33c..05aaf8c17 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -83,7 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, - VectorFilterUnknownSuffix(String), + VectorFilterUnknownSuffix(Option<&'static str>, String), VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, @@ -214,16 +214,23 @@ impl Display for Error<'_> { ErrorKind::VectorFilterLeftover => { writeln!(f, "The vector filter has leftover tokens.")? } - ErrorKind::VectorFilterUnknownSuffix(value) if value.as_str() == "." => { + ErrorKind::VectorFilterUnknownSuffix(_, value) if value.as_str() == "." => { writeln!(f, "Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.")?; } - ErrorKind::VectorFilterUnknownSuffix(value) => { + ErrorKind::VectorFilterUnknownSuffix(None, value) if ["fragments", "userProvided", "documentTemplate", "regenerate"].contains(&value.as_str()) => { + // This will happen with "_vectors.rest.\"userProvided\"" for instance + writeln!(f, "Was expecting this part to be unquoted.")? + } + ErrorKind::VectorFilterUnknownSuffix(None, value) => { if let Some(suggestion) = key_suggestion(value, &["fragments", "userProvided", "documentTemplate", "regenerate"]) { writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`. Did you mean `{suggestion}`?")?; } else { writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`.")?; } } + ErrorKind::VectorFilterUnknownSuffix(Some(previous_filter_kind), value) => { + writeln!(f, "Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `{previous_filter_kind}` and `{value}`.")? + }, ErrorKind::VectorFilterInvalidFragment => { writeln!(f, "The vector filter's fragment is invalid.")? } @@ -234,7 +241,7 @@ impl Display for Error<'_> { writeln!(f, "Was expecting embedder name but found nothing.")? } ErrorKind::VectorFilterInvalidEmbedder => { - writeln!(f, "The vector filter's embedder is invalid.")? + writeln!(f, "The vector filter's embedder name is invalid.")? } ErrorKind::VectorFilterOperation => { writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 75cbecc26..8f6f02691 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -1018,7 +1018,7 @@ pub mod tests { 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" - The vector filter's embedder is invalid. + The vector filter's embedder name is invalid. 30:30 _vectors."embedderName EXISTS "#); insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" @@ -1057,6 +1057,14 @@ pub mod tests { Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `fargments`. Did you mean `fragments`? 23:32 _vectors.embedderName.fargments.test EXISTS "); + insta::assert_snapshot!(p(r#"_vectors.embedderName."userProvided" EXISTS"#), @r#" + Was expecting this part to be unquoted. + 24:36 _vectors.embedderName."userProvided" EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors.embedderName.userProvided.fragments.test EXISTS"#), @r" + Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `userProvided` and `fragments`. + 36:45 _vectors.embedderName.userProvided.fragments.test EXISTS + "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. From cdeca595872bb8fa668f0c2fc2f9e714f3da5294 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 17:14:36 +0200 Subject: [PATCH 75/76] Add error message for quoting errors --- crates/filter-parser/src/condition.rs | 5 +++-- crates/filter-parser/src/error.rs | 6 +++++- crates/filter-parser/src/lib.rs | 8 ++++---- crates/filter-parser/src/value.rs | 12 ++++++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index c407a1e45..8e3c04040 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -19,6 +19,7 @@ use Condition::*; use crate::error::IResultExt; use crate::value::parse_vector_value; +use crate::value::parse_vector_value_cut; use crate::Error; use crate::ErrorKind; use crate::VectorFilter; @@ -141,13 +142,13 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> } let (input, embedder_name) = - parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidEmbedder)?; let (input, filter) = alt(( map( preceded(tag(".fragments"), |input| { let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?; - parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidFragment) + parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidFragment) }), VectorFilter::Fragment, ), diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 05aaf8c17..e381f45e2 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -79,6 +79,7 @@ pub enum ErrorKind<'a> { MisusedGeoRadius, MisusedGeoBoundingBox, VectorFilterLeftover, + VectorFilterInvalidQuotes, VectorFilterMissingEmbedder, VectorFilterInvalidEmbedder, VectorFilterMissingFragment, @@ -232,7 +233,7 @@ impl Display for Error<'_> { writeln!(f, "Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `{previous_filter_kind}` and `{value}`.")? }, ErrorKind::VectorFilterInvalidFragment => { - writeln!(f, "The vector filter's fragment is invalid.")? + writeln!(f, "The vector filter's fragment name is invalid.")? } ErrorKind::VectorFilterMissingFragment => { writeln!(f, "The vector filter is missing a fragment name.")? @@ -246,6 +247,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterOperation => { writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? } + ErrorKind::VectorFilterInvalidQuotes => { + writeln!(f, "The quotes in one of the values are inconsistent.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 8f6f02691..c761c583b 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -1018,8 +1018,8 @@ pub mod tests { 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" - The vector filter's embedder name is invalid. - 30:30 _vectors."embedderName EXISTS + The quotes in one of the values are inconsistent. + 10:30 _vectors."embedderName EXISTS "#); insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" The vector filter has leftover tokens. @@ -1034,7 +1034,7 @@ pub mod tests { 32:39 _vectors.embedderName.fragments EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. EXISTS"#), @r" - The vector filter's fragment is invalid. + The vector filter's fragment name is invalid. 33:40 _vectors.embedderName.fragments. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" @@ -1042,7 +1042,7 @@ pub mod tests { 38:49 _vectors.embedderName.fragments.test test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" - The vector filter's fragment is invalid. + The vector filter's fragment name is invalid. 33:45 _vectors.embedderName.fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index dac96b4f4..35a5c0ab4 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -113,6 +113,18 @@ pub fn parse_vector_value(input: Span) -> IResult { } } +pub fn parse_vector_value_cut<'a>(input: Span<'a>, kind: ErrorKind<'a>) -> IResult<'a, Token<'a>> { + parse_vector_value(input).map_err(|e| match e { + nom::Err::Failure(e) => match e.kind() { + ErrorKind::Char(c) if *c == '"' || *c == '\'' => { + crate::Error::failure_from_kind(input, ErrorKind::VectorFilterInvalidQuotes) + } + _ => crate::Error::failure_from_kind(input, kind), + }, + _ => crate::Error::failure_from_kind(input, kind), + }) +} + /// value = WS* ( word | singleQuoted | doubleQuoted) WS+ pub fn parse_value(input: Span) -> IResult { // to get better diagnostic message we are going to strip the left whitespaces from the input right now From 307ea38c2a8e3c08e5da6734f280a583388a653a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 17:19:37 +0200 Subject: [PATCH 76/76] Remove old irrelevant tests --- crates/meilisearch/tests/search/filters.rs | 35 ---------------------- 1 file changed, 35 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 9cd6575ac..ef562bf4f 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -966,26 +966,6 @@ async fn vector_filter_non_existant_fragment() { "#); } -#[actix_rt::test] -async fn vector_filter_specific_fragment_user_provided() { - let index = shared_index_for_fragments().await; - - let (value, _code) = index - .search_post(json!({ - "filter": "_vectors.rest.fragments.other.userProvided EXISTS", - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "message": "The vector filter has leftover tokens.\n30:50 _vectors.rest.fragments.other.userProvided EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); -} - #[actix_rt::test] async fn vector_filter_document_template_but_fragments_used() { let index = shared_index_for_fragments().await; @@ -1179,19 +1159,4 @@ async fn vector_filter_regenerate() { "estimatedTotalHits": 3 } "#); - - let (value, _code) = index - .search_post(json!({ - "filter": format!("_vectors.rest.fragments.basic.regenerate EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "message": "The vector filter has leftover tokens.\n30:48 _vectors.rest.fragments.basic.regenerate EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); }