diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index 2062e1c28..a951a7ca6 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -62,13 +62,14 @@ impl IndexScheduler { let ExportIndexSettings { filter, override_settings } = export_settings; let index = self.index(uid)?; let index_rtxn = index.read_txn()?; + let bearer = api_key.map(|api_key| format!("Bearer {api_key}")); // First, check if the index already exists let url = format!("{base_url}/indexes/{uid}"); let response = retry(&must_stop_processing, || { let mut request = agent.get(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(Default::default()).map_err(into_backoff_error) @@ -90,8 +91,8 @@ impl IndexScheduler { let url = format!("{base_url}/indexes"); retry(&must_stop_processing, || { let mut request = agent.post(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } let index_param = json!({ "uid": uid, "primaryKey": primary_key }); request.send_json(&index_param).map_err(into_backoff_error) @@ -103,8 +104,8 @@ impl IndexScheduler { let url = format!("{base_url}/indexes/{uid}"); retry(&must_stop_processing, || { let mut request = agent.patch(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } let index_param = json!({ "primaryKey": primary_key }); request.send_json(&index_param).map_err(into_backoff_error) @@ -122,7 +123,6 @@ impl IndexScheduler { } // Retry logic for sending settings let url = format!("{base_url}/indexes/{uid}/settings"); - let bearer = api_key.map(|api_key| format!("Bearer {api_key}")); retry(&must_stop_processing, || { let mut request = agent.patch(&url); if let Some(bearer) = bearer.as_ref() { @@ -167,10 +167,10 @@ impl IndexScheduler { }, ); - let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB + let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024); // defaults to 20 MiB let documents_url = format!("{base_url}/indexes/{uid}/documents"); - request_threads() + let results = request_threads() .broadcast(|ctx| { let index_rtxn = index .read_txn() @@ -265,9 +265,8 @@ impl IndexScheduler { let mut request = agent.post(&documents_url); request = request.set("Content-Type", "application/x-ndjson"); request = request.set("Content-Encoding", "gzip"); - if let Some(api_key) = api_key { - request = request - .set("Authorization", &(format!("Bearer {api_key}"))); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(&compressed_buffer).map_err(into_backoff_error) })?; @@ -276,7 +275,7 @@ impl IndexScheduler { } buffer.extend_from_slice(&tmp_buffer); - if i % 100 == 0 { + if i > 0 && i % 100 == 0 { step.fetch_add(100, atomic::Ordering::Relaxed); } } @@ -284,8 +283,8 @@ impl IndexScheduler { retry(&must_stop_processing, || { let mut request = agent.post(&documents_url); request = request.set("Content-Type", "application/x-ndjson"); - if let Some(api_key) = api_key { - request = request.set("Authorization", &(format!("Bearer {api_key}"))); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(&buffer).map_err(into_backoff_error) })?; @@ -298,6 +297,9 @@ impl IndexScheduler { Some(uid.to_string()), ) })?; + for result in results { + result?; + } step.store(total_documents, atomic::Ordering::Relaxed); } diff --git a/crates/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs index 27d163192..6f5a5c2a2 100644 --- a/crates/meilisearch-auth/src/lib.rs +++ b/crates/meilisearch-auth/src/lib.rs @@ -158,7 +158,7 @@ impl AuthController { self.store.delete_all_keys() } - /// Delete all the keys in the DB. + /// Insert a key directly into the store. pub fn raw_insert_key(&mut self, key: Key) -> Result<()> { self.store.put_api_key(key)?; Ok(()) @@ -351,6 +351,7 @@ pub struct IndexSearchRules { fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_chat())?; + store.put_api_key(Key::default_read_only_admin())?; store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index bae27afe4..eb2170f08 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -88,7 +88,13 @@ impl HeedAuthStore { let mut actions = HashSet::new(); for action in &key.actions { match action { - Action::All => actions.extend(enum_iterator::all::()), + Action::All => { + actions.extend(enum_iterator::all::()); + actions.remove(&Action::AllGet); + } + Action::AllGet => { + actions.extend(enum_iterator::all::().filter(|a| a.is_read())) + } Action::DocumentsAll => { actions.extend( [Action::DocumentsGet, Action::DocumentsDelete, Action::DocumentsAdd] diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 3ba31c2cb..e210f8df3 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -144,6 +144,21 @@ impl Key { } } + pub fn default_read_only_admin() -> Self { + let now = OffsetDateTime::now_utc(); + let uid = Uuid::new_v4(); + Self { + name: Some("Default Read-Only Admin API Key".to_string()), + description: Some("Use it to read information across the whole database. Caution! Do not expose this key on a public frontend".to_string()), + uid, + actions: vec![Action::AllGet, Action::KeysGet], + indexes: vec![IndexUidPattern::all()], + expires_at: None, + created_at: now, + updated_at: now, + } + } + pub fn default_search() -> Self { let now = OffsetDateTime::now_utc(); let uid = Uuid::new_v4(); @@ -218,6 +233,9 @@ pub enum Action { #[serde(rename = "*")] #[deserr(rename = "*")] All = 0, + #[serde(rename = "*.get")] + #[deserr(rename = "*.get")] + AllGet, #[serde(rename = "search")] #[deserr(rename = "search")] Search, @@ -399,6 +417,52 @@ impl Action { } } + /// Whether the action should be included in [Action::AllRead]. + pub fn is_read(&self) -> bool { + use Action::*; + + // It's using an exhaustive match to force the addition of new actions. + match self { + // Any action that expands to others must return false, as it wouldn't be able to expand recursively. + All | AllGet | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll + | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, + + Search => true, + DocumentsAdd => false, + DocumentsGet => true, + DocumentsDelete => false, + Export => true, + IndexesAdd => false, + IndexesGet => true, + IndexesUpdate => false, + IndexesDelete => false, + IndexesSwap => false, + TasksCancel => false, + TasksDelete => false, + TasksGet => true, + SettingsGet => true, + SettingsUpdate => false, + StatsGet => true, + MetricsGet => true, + DumpsCreate => false, + SnapshotsCreate => false, + Version => true, + KeysAdd => false, + KeysGet => false, // Disabled in order to prevent privilege escalation + KeysUpdate => false, + KeysDelete => false, + ExperimentalFeaturesGet => true, + ExperimentalFeaturesUpdate => false, + NetworkGet => true, + NetworkUpdate => false, + ChatCompletions => false, // Disabled because it might trigger generation of new chats + ChatsGet => true, + ChatsDelete => false, + ChatsSettingsGet => true, + ChatsSettingsUpdate => false, + } + } + pub const fn repr(&self) -> u8 { *self as u8 } @@ -408,6 +472,7 @@ pub mod actions { use super::Action::*; pub(crate) const ALL: u8 = All.repr(); + pub const ALL_GET: u8 = AllGet.repr(); pub const SEARCH: u8 = Search.repr(); pub const DOCUMENTS_ALL: u8 = DocumentsAll.repr(); pub const DOCUMENTS_ADD: u8 = DocumentsAdd.repr(); diff --git a/crates/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs index 91c6c23fa..8d4430f07 100644 --- a/crates/meilisearch/src/error.rs +++ b/crates/meilisearch/src/error.rs @@ -49,7 +49,7 @@ pub enum MeilisearchHttpError { TooManySearchRequests(usize), #[error("Internal error: Search limiter is down.")] SearchLimiterIsDown, - #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(*.0 as u64).get_appropriate_unit(UnitType::Binary))] + #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(*.0 as u64).get_appropriate_unit(if *.0 % 1024 == 0 { UnitType::Binary } else { UnitType::Decimal }))] PayloadTooLarge(usize), #[error("Two indexes must be given for each swap. The list `[{}]` contains {} indexes.", .0.iter().map(|uid| format!("\"{uid}\"")).collect::>().join(", "), .0.len() diff --git a/crates/meilisearch/src/routes/export_analytics.rs b/crates/meilisearch/src/routes/export_analytics.rs index b66a5133b..a2f0a129d 100644 --- a/crates/meilisearch/src/routes/export_analytics.rs +++ b/crates/meilisearch/src/routes/export_analytics.rs @@ -1,3 +1,5 @@ +use url::Url; + use crate::analytics::Aggregate; use crate::routes::export::Export; @@ -5,6 +7,7 @@ use crate::routes::export::Export; pub struct ExportAnalytics { total_received: usize, has_api_key: bool, + sum_exports_meilisearch_cloud: usize, sum_index_patterns: usize, sum_patterns_with_filter: usize, sum_patterns_with_override_settings: usize, @@ -13,8 +16,14 @@ pub struct ExportAnalytics { impl ExportAnalytics { pub fn from_export(export: &Export) -> Self { - let Export { url: _, api_key, payload_size, indexes } = export; + let Export { url, api_key, payload_size, indexes } = export; + let url = Url::parse(url).ok(); + let is_meilisearch_cloud = url.as_ref().and_then(Url::host_str).is_some_and(|host| { + host.ends_with("meilisearch.dev") + || host.ends_with("meilisearch.com") + || host.ends_with("meilisearch.io") + }); let has_api_key = api_key.is_some(); let index_patterns_count = indexes.as_ref().map_or(0, |indexes| indexes.len()); let patterns_with_filter_count = indexes.as_ref().map_or(0, |indexes| { @@ -33,6 +42,7 @@ impl ExportAnalytics { Self { total_received: 1, has_api_key, + sum_exports_meilisearch_cloud: is_meilisearch_cloud as usize, sum_index_patterns: index_patterns_count, sum_patterns_with_filter: patterns_with_filter_count, sum_patterns_with_override_settings: patterns_with_override_settings_count, @@ -49,6 +59,7 @@ impl Aggregate for ExportAnalytics { fn aggregate(mut self: Box, other: Box) -> Box { self.total_received += other.total_received; self.has_api_key |= other.has_api_key; + self.sum_exports_meilisearch_cloud += other.sum_exports_meilisearch_cloud; self.sum_index_patterns += other.sum_index_patterns; self.sum_patterns_with_filter += other.sum_patterns_with_filter; self.sum_patterns_with_override_settings += other.sum_patterns_with_override_settings; @@ -63,6 +74,12 @@ impl Aggregate for ExportAnalytics { Some(self.payload_sizes.iter().sum::() / self.payload_sizes.len() as u64) }; + let avg_exports_meilisearch_cloud = if self.total_received == 0 { + None + } else { + Some(self.sum_exports_meilisearch_cloud as f64 / self.total_received as f64) + }; + let avg_index_patterns = if self.total_received == 0 { None } else { @@ -84,6 +101,7 @@ impl Aggregate for ExportAnalytics { serde_json::json!({ "total_received": self.total_received, "has_api_key": self.has_api_key, + "avg_exports_meilisearch_cloud": avg_exports_meilisearch_cloud, "avg_index_patterns": avg_index_patterns, "avg_patterns_with_filter": avg_patterns_with_filter, "avg_patterns_with_override_settings": avg_patterns_with_override_settings, diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 2688dd918..0b8a3d2c5 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -419,14 +419,14 @@ async fn error_add_api_key_invalid_parameters_actions() { let (response, code) = server.add_api_key(content).await; meili_snap::snapshot!(code, @"400 Bad Request"); - meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" } - "###); + "#); } #[actix_rt::test] @@ -790,7 +790,7 @@ async fn list_api_keys() { meili_snap::snapshot!(code, @"201 Created"); let (response, code) = server.list_api_keys("").await; - meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r#" { "results": [ { @@ -850,6 +850,22 @@ async fn list_api_keys() { "createdAt": "[ignored]", "updatedAt": "[ignored]" }, + { + "name": "Default Read-Only Admin API Key", + "description": "Use it to read information across the whole database. Caution! Do not expose this key on a public frontend", + "key": "[ignored]", + "uid": "[ignored]", + "actions": [ + "*.get", + "keys.get" + ], + "indexes": [ + "*" + ], + "expiresAt": null, + "createdAt": "[ignored]", + "updatedAt": "[ignored]" + }, { "name": "Default Chat API Key", "description": "Use it to chat and search from the frontend", @@ -869,9 +885,9 @@ async fn list_api_keys() { ], "offset": 0, "limit": 20, - "total": 4 + "total": 5 } - "###); + "#); meili_snap::snapshot!(code, @"200 OK"); } diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index 687cb67a0..e8d935fde 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -91,14 +91,14 @@ async fn create_api_key_bad_actions() { // can't parse let (response, code) = server.add_api_key(json!({ "actions": ["doggo"] })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 4367650c5..aa4501636 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -97,6 +97,7 @@ impl Server { self.use_api_key(master_key); let (response, code) = self.list_api_keys("").await; assert_eq!(200, code, "{:?}", response); + // TODO: relying on the order of keys is not ideal, we should use the name instead let admin_key = &response["results"][1]["key"]; self.use_api_key(admin_key.as_str().unwrap()); } diff --git a/crates/meilisearch/tests/index/stats.rs b/crates/meilisearch/tests/index/stats.rs index 610601318..7f2ca9b4a 100644 --- a/crates/meilisearch/tests/index/stats.rs +++ b/crates/meilisearch/tests/index/stats.rs @@ -1,5 +1,4 @@ use crate::common::{shared_does_not_exists_index, Server}; - use crate::json; #[actix_rt::test] diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index 47e699380..f50f7f940 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -692,3 +692,68 @@ async fn granular_filterable_attributes() { ] "###); } + +#[actix_rt::test] +async fn test_searchable_attributes_order() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // 1) Create an index with settings "searchableAttributes": ["title", "overview"] + let (response, code) = index.create(None).await; + assert_eq!(code, 202, "{response}"); + server.wait_task(response.uid()).await.succeeded(); + + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["title", "overview"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 2) Add documents in the index + let documents = json!([ + { + "id": 1, + "title": "The Matrix", + "overview": "A computer hacker learns from mysterious rebels about the true nature of his reality." + }, + { + "id": 2, + "title": "Inception", + "overview": "A thief who steals corporate secrets through dream-sharing technology." + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202, "{response}"); + server.wait_task(response.uid()).await.succeeded(); + + // 3) Modify the settings "searchableAttributes": ["overview", "title"] (overview is put first) + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["overview", "title"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 4) Check if it has been applied + let (response, code) = index.settings().await; + assert_eq!(code, 200, "{response}"); + assert_eq!(response["searchableAttributes"], json!(["overview", "title"])); + + // 5) Re-modify the settings "searchableAttributes": ["title", "overview"] (title is put first) + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["title", "overview"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 6) Check if it has been applied + let (response, code) = index.settings().await; + assert_eq!(code, 200, "{response}"); + assert_eq!(response["searchableAttributes"], json!(["title", "overview"])); +} diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap index af7e82c8b..3c97dbe70 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap @@ -61,7 +61,16 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "pagination": { "maxTotalHits": 15 }, - "embedders": {}, + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "pooling": "forceMean", + "documentTemplate": "{{doc.description}}", + "documentTemplateMaxBytes": 400 + } + }, "searchCutoffMs": 8000, "localizedAttributes": [ { diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap new file mode 100644 index 000000000..5baf8155c --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap @@ -0,0 +1,40 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +[ + { + "id": 1, + "name": "kefir", + "surname": [ + "kef", + "kefkef", + "kefirounet", + "boubou" + ], + "age": 1.4, + "description": "kefir est un petit chien blanc très mignon", + "_vectors": { + "doggo_embedder": { + "embeddings": "[vector]", + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "surname": [ + "untel", + "tétel", + "iouiou" + ], + "age": 11.5, + "description": "intel est un grand beagle très mignon", + "_vectors": { + "doggo_embedder": { + "embeddings": "[vector]", + "regenerate": false + } + } + } +] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index fb62b35da..81b50fb92 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -642,8 +791,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "batchStrategy": "unspecified" } ], - "total": 25, + "total": 31, "limit": 1000, - "from": 24, + "from": 30, "next": null } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index abb4dcdd9..1ec334fed 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -497,8 +609,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "2025-01-16T16:45:16.131303739Z" } ], - "total": 26, + "total": 32, "limit": 1000, - "from": 25, + "from": 31, "next": null } diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb index 4c80ffe2c..80fb2b9d5 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb index c31db3415..95ca0a9da 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb index c99608b77..5fa5e6b49 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb index 226be2332..f2bcb1b8b 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb index 6d38eab08..b8e0e358d 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs index 1b2ae054c..b98f27b2d 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -114,13 +114,13 @@ async fn check_the_index_scheduler(server: &Server) { // All the indexes are still present let (indexes, _) = server.list_indexes(None, None).await; - snapshot!(indexes, @r#" + snapshot!(indexes, @r###" { "results": [ { "uid": "kefir", "createdAt": "2025-01-16T16:45:16.020663157Z", - "updatedAt": "2025-01-23T11:36:22.634859166Z", + "updatedAt": "2025-07-07T13:43:08.835381Z", "primaryKey": "id" } ], @@ -128,7 +128,7 @@ async fn check_the_index_scheduler(server: &Server) { "limit": 20, "total": 1 } - "#); + "###); // And their metadata are still right let (stats, _) = server.stats().await; assert_json_snapshot!(stats, { @@ -141,21 +141,21 @@ async fn check_the_index_scheduler(server: &Server) { { "databaseSize": "[bytes]", "usedDatabaseSize": "[bytes]", - "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "lastUpdate": "2025-07-07T13:43:08.835381Z", "indexes": { "kefir": { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } } @@ -227,21 +227,21 @@ async fn check_the_index_scheduler(server: &Server) { { "databaseSize": "[bytes]", "usedDatabaseSize": "[bytes]", - "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "lastUpdate": "2025-07-07T13:43:08.835381Z", "indexes": { "kefir": { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } } @@ -254,18 +254,18 @@ async fn check_the_index_scheduler(server: &Server) { ".avgDocumentSize" => "[bytes]", }), @r###" { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } "###); @@ -295,4 +295,8 @@ async fn check_the_index_features(server: &Server) { let (results, _status) = kefir.search_post(json!({ "sort": ["age:asc"], "filter": "surname = kefirounet" })).await; snapshot!(results, name: "search_with_sort_and_filter"); + + // ensuring we can get the vectors and their `regenerate` is still good. + let (results, _status) = kefir.search_post(json!({"retrieveVectors": true})).await; + snapshot!(json_string!(results["hits"], {"[]._vectors.doggo_embedder.embeddings" => "[vector]"}), name: "search_with_retrieve_vectors"); } diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 911f51865..d2f74da2a 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -554,10 +554,10 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { match self.searchable_fields { Setting::Set(ref fields) => { // Check to see if the searchable fields changed before doing anything else - let old_fields = self.index.searchable_fields(self.wtxn)?; + let old_fields = self.index.user_defined_searchable_fields(self.wtxn)?; let did_change = { let new_fields = fields.iter().map(String::as_str).collect::>(); - new_fields != old_fields + old_fields.is_none_or(|old| new_fields != old) }; if !did_change { return Ok(false); diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 9f64ca0e3..f53319a37 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -2,6 +2,7 @@ mod v1_12; mod v1_13; mod v1_14; mod v1_15; +mod v1_16; use heed::RwTxn; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; @@ -10,6 +11,7 @@ use v1_15::Latest_V1_14_To_Latest_V1_15; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; +use crate::update::upgrade::v1_16::Latest_V1_15_To_V1_16_0; use crate::{Index, InternalError, Result}; trait UpgradeIndex { @@ -24,6 +26,59 @@ trait UpgradeIndex { fn target_version(&self) -> (u32, u32, u32); } +const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ + &V1_12_To_V1_12_3 {}, + &V1_12_3_To_V1_13_0 {}, + &V1_13_0_To_V1_13_1 {}, + &V1_13_1_To_Latest_V1_13 {}, + &Latest_V1_13_To_Latest_V1_14 {}, + &Latest_V1_14_To_Latest_V1_15 {}, + &Latest_V1_15_To_V1_16_0 {}, + // This is the last upgrade function, it will be called when the index is up to date. + // any other upgrade function should be added before this one. + &ToCurrentNoOp {}, +]; + +/// Causes a compile-time error if the argument is not in range of `0..UPGRADE_FUNCTIONS.len()` +macro_rules! function_index { + ($start:expr) => {{ + const _CHECK_INDEX: () = { + if $start >= $crate::update::upgrade::UPGRADE_FUNCTIONS.len() { + panic!("upgrade functions out of range") + } + }; + + $start + }}; +} + +const fn start(from: (u32, u32, u32)) -> Option { + let start = match from { + (1, 12, 0..=2) => function_index!(0), + (1, 12, 3..) => function_index!(1), + (1, 13, 0) => function_index!(2), + (1, 13, _) => function_index!(4), + (1, 14, _) => function_index!(5), + // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. + (1, 15, _) => function_index!(6), + (1, 16, _) => function_index!(7), + // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually + // considering dumpless upgrade. + (_major, _minor, _patch) => return None, + }; + + Some(start) +} + +/// Causes a compile-time error if the latest package cannot be upgraded. +/// +/// This serves as a reminder to consider the proper dumpless upgrade implementation when changing the package version. +const _CHECK_PACKAGE_CAN_UPGRADE: () = { + if start((VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)).is_none() { + panic!("cannot upgrade from latest package version") + } +}; + /// Return true if the cached stats of the index must be regenerated pub fn upgrade( wtxn: &mut RwTxn, @@ -36,33 +91,12 @@ where MSP: Fn() -> bool + Sync, { let from = index.get_version(wtxn)?.unwrap_or(db_version); - let upgrade_functions: &[&dyn UpgradeIndex] = &[ - &V1_12_To_V1_12_3 {}, - &V1_12_3_To_V1_13_0 {}, - &V1_13_0_To_V1_13_1 {}, - &V1_13_1_To_Latest_V1_13 {}, - &Latest_V1_13_To_Latest_V1_14 {}, - &Latest_V1_14_To_Latest_V1_15 {}, - // This is the last upgrade function, it will be called when the index is up to date. - // any other upgrade function should be added before this one. - &ToCurrentNoOp {}, - ]; - let start = match from { - (1, 12, 0..=2) => 0, - (1, 12, 3..) => 1, - (1, 13, 0) => 2, - (1, 13, _) => 4, - (1, 14, _) => 5, - // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. - (1, 15, _) => 6, - (major, minor, patch) => { - return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) - } - }; + let start = + start(from).ok_or_else(|| InternalError::CannotUpgradeToVersion(from.0, from.1, from.2))?; enum UpgradeVersion {} - let upgrade_path = &upgrade_functions[start..]; + let upgrade_path = &UPGRADE_FUNCTIONS[start..]; let mut current_version = from; let mut regenerate_stats = false; diff --git a/crates/milli/src/update/upgrade/v1_15.rs b/crates/milli/src/update/upgrade/v1_15.rs index cea4783a1..3457e69ba 100644 --- a/crates/milli/src/update/upgrade/v1_15.rs +++ b/crates/milli/src/update/upgrade/v1_15.rs @@ -1,4 +1,6 @@ use heed::RwTxn; +use roaring::RoaringBitmap; +use serde::Deserialize; use super::UpgradeIndex; use crate::progress::Progress; @@ -26,3 +28,14 @@ impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 { (1, 15, 0) } } + +/// Parts of v1.15 `IndexingEmbeddingConfig` that are relevant for upgrade to v1.16 +/// +/// # Warning +/// +/// This object should not be rewritten to the DB, only read to get the name and `user_provided` roaring. +#[derive(Debug, Deserialize)] +pub struct IndexEmbeddingConfig { + pub name: String, + pub user_provided: RoaringBitmap, +} diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs new file mode 100644 index 000000000..f43efd77d --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -0,0 +1,48 @@ +use heed::types::{SerdeJson, Str}; +use heed::RwTxn; + +use super::UpgradeIndex; +use crate::progress::Progress; +use crate::vector::db::{EmbedderInfo, EmbeddingStatus}; +use crate::{Index, InternalError, Result}; + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_15_To_V1_16_0(); + +impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + let v1_15_indexing_configs = index + .main + .remap_types::>>() + .get(wtxn, crate::index::main_key::EMBEDDING_CONFIGS)? + .unwrap_or_default(); + + let embedders = index.embedding_configs(); + for config in v1_15_indexing_configs { + let embedder_id = embedders.embedder_id(wtxn, &config.name)?.ok_or( + InternalError::DatabaseMissingEntry { + db_name: crate::index::db_name::VECTOR_EMBEDDER_CATEGORY_ID, + key: None, + }, + )?; + let info = EmbedderInfo { + embedder_id, + // v1.15 used not to make a difference between `user_provided` and `! regenerate`. + embedding_status: EmbeddingStatus::from_user_provided(config.user_provided), + }; + embedders.put_embedder_info(wtxn, &config.name, &info)?; + } + + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 16, 0) + } +} diff --git a/crates/milli/src/vector/db.rs b/crates/milli/src/vector/db.rs index 0e890fac9..2fea75d68 100644 --- a/crates/milli/src/vector/db.rs +++ b/crates/milli/src/vector/db.rs @@ -117,6 +117,13 @@ impl EmbeddingStatus { Default::default() } + /// Create a new `EmbeddingStatus` that assumes that any `user_provided` docid is also skipping regenerate. + /// + /// Used for migration from v1.15 and earlier DBs. + pub(crate) fn from_user_provided(user_provided: RoaringBitmap) -> Self { + Self { user_provided, skip_regenerate_different_from_user_provided: Default::default() } + } + /// Whether the document contains user-provided vectors for that embedder. pub fn is_user_provided(&self, docid: DocumentId) -> bool { self.user_provided.contains(docid)