From 0f1c78b185a665475701ef28806006691b2994de Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Fri, 1 Aug 2025 23:17:19 +0200 Subject: [PATCH 01/38] Add index rename feature --- .../index-scheduler/src/index_mapper/mod.rs | 14 ++++++++ crates/index-scheduler/src/processing.rs | 6 ++++ .../src/scheduler/autobatcher.rs | 14 +++++++- .../src/scheduler/create_batch.rs | 20 ++++++++++- .../src/scheduler/process_batch.rs | 16 ++++++++- crates/meilisearch-types/src/tasks.rs | 34 ++++++++++++++++++- 6 files changed, 100 insertions(+), 4 deletions(-) diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index e6bdccd41..d578b03dd 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -526,6 +526,20 @@ impl IndexMapper { Ok(()) } + /// Rename an index. + pub fn rename(&self, wtxn: &mut RwTxn, current: &str, new: &str) -> Result<()> { + let uuid = self + .index_mapping + .get(wtxn, current)? + .ok_or_else(|| Error::IndexNotFound(current.to_string()))?; + if self.index_mapping.get(wtxn, new)?.is_some() { + return Err(Error::IndexAlreadyExists(new.to_string())); + } + self.index_mapping.delete(wtxn, current)?; + self.index_mapping.put(wtxn, new, &uuid)?; + Ok(()) + } + /// The stats of an index. /// /// If available in the cache, they are directly returned. diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index 3da81f143..84b0a5360 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -125,6 +125,12 @@ make_enum_progress! { } } +make_enum_progress! { + pub enum RenameIndexProgress { + RenamingTheIndex, + } +} + make_enum_progress! { pub enum DeleteIndexProgress { DeletingTheIndex, diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index b3f7d2743..6b62d36e0 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -24,6 +24,7 @@ enum AutobatchKind { IndexCreation, IndexDeletion, IndexUpdate, + IndexRename, IndexSwap, } @@ -67,6 +68,7 @@ impl From for AutobatchKind { KindWithContent::IndexDeletion { .. } => AutobatchKind::IndexDeletion, KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation, KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate, + KindWithContent::IndexRename { .. } => AutobatchKind::IndexRename, KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap, KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } @@ -115,6 +117,9 @@ pub enum BatchKind { IndexUpdate { id: TaskId, }, + IndexRename { + id: TaskId, + }, IndexSwap { id: TaskId, }, @@ -176,6 +181,13 @@ impl BatchKind { )), false, ), + K::IndexRename => ( + Break(( + BatchKind::IndexRename { id: task_id }, + BatchStopReason::TaskCannotBeBatched { kind, id: task_id }, + )), + false, + ), K::IndexSwap => ( Break(( BatchKind::IndexSwap { id: task_id }, @@ -288,7 +300,7 @@ impl BatchKind { match (self, autobatch_kind) { // We don't batch any of these operations - (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), + (this, K::IndexCreation | K::IndexUpdate | K::IndexRename | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists. (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => { Break((this, BatchStopReason::IndexCreationMismatch { id })) diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index e78ed2c2e..fa84200a1 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -40,6 +40,11 @@ pub(crate) enum Batch { primary_key: Option, task: Task, }, + IndexRename { + index_uid: String, + new_index_uid: String, + task: Task, + }, IndexDeletion { index_uid: String, tasks: Vec, @@ -108,7 +113,8 @@ impl Batch { | Batch::Dump(task) | Batch::IndexCreation { task, .. } | Batch::Export { task } - | Batch::IndexUpdate { task, .. } => { + | Batch::IndexUpdate { task, .. } + | Batch::IndexRename { task, .. } => { RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap() } Batch::SnapshotCreation(tasks) @@ -153,6 +159,7 @@ impl Batch { IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } + | IndexRename { index_uid, .. } | IndexDeletion { index_uid, .. } => Some(index_uid), } } @@ -171,6 +178,7 @@ impl fmt::Display for Batch { Batch::IndexOperation { op, .. } => write!(f, "{op}")?, Batch::IndexCreation { .. } => f.write_str("IndexCreation")?, Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?, + Batch::IndexRename { .. } => f.write_str("IndexRename")?, Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?, Batch::IndexSwap { .. } => f.write_str("IndexSwap")?, Batch::Export { .. } => f.write_str("Export")?, @@ -411,6 +419,16 @@ impl IndexScheduler { }; Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task })) } + BatchKind::IndexRename { id } => { + let mut task = + self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; + current_batch.processing(Some(&mut task)); + let (new_uid) = match &task.kind { + KindWithContent::IndexRename { new_index_uid, .. } => new_index_uid.clone(), + _ => unreachable!(), + }; + Ok(Some(Batch::IndexRename { index_uid, new_index_uid: new_uid, task })) + } BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion { index_uid, index_has_been_created: must_create_index, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index c21ab27ad..d801287a7 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -15,7 +15,7 @@ use super::create_batch::Batch; use crate::processing::{ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, - UpdateIndexProgress, + UpdateIndexProgress, RenameIndexProgress, }; use crate::utils::{ self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task, @@ -229,6 +229,20 @@ impl IndexScheduler { progress, ) } + Batch::IndexRename { index_uid, new_index_uid, mut task } => { + progress.update_progress(RenameIndexProgress::RenamingTheIndex); + let mut wtxn = self.env.write_txn()?; + self.index_mapper.rename(&mut wtxn, &index_uid, &new_index_uid)?; + self.queue.tasks.update_index(&mut wtxn, &new_index_uid, |bm| { + let old = self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); + *bm |= &old; + })?; + self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; + wtxn.commit()?; + task.status = Status::Succeeded; + task.details = Some(Details::IndexRename(IndexRenameDetails { old_uid: index_uid, new_uid: new_index_uid })); + Ok((vec![task], ProcessBatchInfo::default())) + } Batch::IndexUpdate { index_uid, primary_key, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); let rtxn = self.env.read_txn()?; diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 99b04f1e3..64826c693 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -141,6 +141,10 @@ pub enum KindWithContent { index_uid: String, primary_key: Option, }, + IndexRename { + index_uid: String, + new_index_uid: String, + }, IndexSwap { swaps: Vec, }, @@ -174,6 +178,13 @@ pub struct IndexSwap { pub indexes: (String, String), } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct IndexRenameDetails { + pub old_uid: String, + pub new_uid: String, +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct ExportIndexSettings { @@ -193,6 +204,7 @@ impl KindWithContent { KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, + KindWithContent::IndexRename { .. } => Kind::IndexRename, KindWithContent::IndexSwap { .. } => Kind::IndexSwap, KindWithContent::TaskCancelation { .. } => Kind::TaskCancelation, KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, @@ -222,6 +234,7 @@ impl KindWithContent { | IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } | IndexDeletion { index_uid } => vec![index_uid], + IndexRename { index_uid, new_index_uid } => vec![index_uid, new_index_uid], IndexSwap { swaps } => { let mut indexes = HashSet::<&str>::default(); for swap in swaps { @@ -274,6 +287,12 @@ impl KindWithContent { | KindWithContent::IndexUpdate { primary_key, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone() }) } + KindWithContent::IndexRename { index_uid, new_index_uid } => { + Some(Details::IndexRename { + old_uid: index_uid.clone(), + new_uid: new_index_uid.clone(), + }) + } KindWithContent::IndexSwap { swaps } => { Some(Details::IndexSwap { swaps: swaps.clone() }) } @@ -344,6 +363,12 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, + KindWithContent::IndexRename { index_uid, new_index_uid } => { + Some(Details::IndexRename { + old_uid: index_uid.clone(), + new_uid: new_index_uid.clone(), + }) + } KindWithContent::IndexCreation { primary_key, .. } | KindWithContent::IndexUpdate { primary_key, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone() }) @@ -538,6 +563,7 @@ pub enum Kind { IndexCreation, IndexDeletion, IndexUpdate, + IndexRename, IndexSwap, TaskCancelation, TaskDeletion, @@ -556,7 +582,8 @@ impl Kind { | Kind::SettingsUpdate | Kind::IndexCreation | Kind::IndexDeletion - | Kind::IndexUpdate => true, + | Kind::IndexUpdate + | Kind::IndexRename => true, Kind::IndexSwap | Kind::TaskCancelation | Kind::TaskDeletion @@ -577,6 +604,7 @@ impl Display for Kind { Kind::IndexCreation => write!(f, "indexCreation"), Kind::IndexDeletion => write!(f, "indexDeletion"), Kind::IndexUpdate => write!(f, "indexUpdate"), + Kind::IndexRename => write!(f, "indexRename"), Kind::IndexSwap => write!(f, "indexSwap"), Kind::TaskCancelation => write!(f, "taskCancelation"), Kind::TaskDeletion => write!(f, "taskDeletion"), @@ -595,6 +623,8 @@ impl FromStr for Kind { Ok(Kind::IndexCreation) } else if kind.eq_ignore_ascii_case("indexUpdate") { Ok(Kind::IndexUpdate) + } else if kind.eq_ignore_ascii_case("indexRename") { + Ok(Kind::IndexRename) } else if kind.eq_ignore_ascii_case("indexSwap") { Ok(Kind::IndexSwap) } else if kind.eq_ignore_ascii_case("indexDeletion") { @@ -692,6 +722,7 @@ pub enum Details { IndexSwap { swaps: Vec, }, + IndexRename(IndexRenameDetails), Export { url: String, api_key: Option, @@ -737,6 +768,7 @@ impl Details { Self::SettingsUpdate { .. } | Self::IndexInfo { .. } | Self::Dump { .. } + | Self::IndexRename { .. } | Self::Export { .. } | Self::UpgradeDatabase { .. } | Self::IndexSwap { .. } => (), From ae2d0a67a4f91bb95c40e721b2b486dfce22400a Mon Sep 17 00:00:00 2001 From: Quentin de Quelen Date: Tue, 5 Aug 2025 19:18:05 +0200 Subject: [PATCH 02/38] Enhance index update functionality to support renaming by adding new_uid field. Update related structures and methods to handle the new index UID during updates, ensuring backward compatibility with existing index operations. --- crates/dump/src/lib.rs | 5 +- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +- crates/index-scheduler/src/dump.rs | 3 +- crates/index-scheduler/src/insta_snapshot.rs | 4 +- crates/index-scheduler/src/processing.rs | 6 - .../src/scheduler/autobatcher.rs | 16 +- .../src/scheduler/autobatcher_test.rs | 6 +- .../src/scheduler/create_batch.rs | 29 +- .../src/scheduler/process_batch.rs | 62 +-- crates/index-scheduler/src/utils.rs | 11 +- crates/meilisearch-types/src/task_view.rs | 25 +- crates/meilisearch-types/src/tasks.rs | 65 ++- crates/meilisearch/src/routes/indexes/mod.rs | 10 + crates/meilisearch/tests/index/mod.rs | 1 + .../meilisearch/tests/index/rename_index.rs | 419 ++++++++++++++++++ 15 files changed, 547 insertions(+), 119 deletions(-) create mode 100644 crates/meilisearch/tests/index/rename_index.rs diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index 81ba40944..b4b339f09 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -129,6 +129,7 @@ pub enum KindDump { }, IndexUpdate { primary_key: Option, + new_uid: Option, }, IndexSwap { swaps: Vec, @@ -210,8 +211,8 @@ impl From for KindDump { KindWithContent::IndexCreation { primary_key, .. } => { KindDump::IndexCreation { primary_key } } - KindWithContent::IndexUpdate { primary_key, .. } => { - KindDump::IndexUpdate { primary_key } + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindDump::IndexUpdate { primary_key, new_uid: new_index_uid } } KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps }, KindWithContent::TaskCancelation { query, tasks } => { diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 3a0c8ef0d..790c239d7 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -85,7 +85,7 @@ impl CompatV5ToV6 { v6::Kind::IndexCreation { primary_key } } v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => { - v6::Kind::IndexUpdate { primary_key } + v6::Kind::IndexUpdate { primary_key, new_uid: None } } v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion, v5::tasks::TaskContent::DocumentAddition { @@ -141,7 +141,7 @@ impl CompatV5ToV6 { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key } + v6::Details::IndexInfo { primary_key, new_uid: None } } v5::Details::DocumentDeletion { received_document_ids, diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 1e681c8e8..18c665ca3 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -197,9 +197,10 @@ impl<'a> Dump<'a> { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, }, - KindDump::IndexUpdate { primary_key } => KindWithContent::IndexUpdate { + KindDump::IndexUpdate { primary_key, new_uid } => KindWithContent::IndexUpdate { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, + new_index_uid: new_uid, }, KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps }, KindDump::TaskCancelation { query, tasks } => { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index cb804d9b4..6d72e4b9f 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key } => { - format!("{{ primary_key: {primary_key:?} }}") + Details::IndexInfo { primary_key, new_uid } => { + format!("{{ primary_key: {primary_key:?}, new_uid: {new_uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index 84b0a5360..3da81f143 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -125,12 +125,6 @@ make_enum_progress! { } } -make_enum_progress! { - pub enum RenameIndexProgress { - RenamingTheIndex, - } -} - make_enum_progress! { pub enum DeleteIndexProgress { DeletingTheIndex, diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index 6b62d36e0..a88a9f0bf 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -24,7 +24,6 @@ enum AutobatchKind { IndexCreation, IndexDeletion, IndexUpdate, - IndexRename, IndexSwap, } @@ -68,7 +67,6 @@ impl From for AutobatchKind { KindWithContent::IndexDeletion { .. } => AutobatchKind::IndexDeletion, KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation, KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate, - KindWithContent::IndexRename { .. } => AutobatchKind::IndexRename, KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap, KindWithContent::TaskCancelation { .. } | KindWithContent::TaskDeletion { .. } @@ -117,9 +115,6 @@ pub enum BatchKind { IndexUpdate { id: TaskId, }, - IndexRename { - id: TaskId, - }, IndexSwap { id: TaskId, }, @@ -181,13 +176,6 @@ impl BatchKind { )), false, ), - K::IndexRename => ( - Break(( - BatchKind::IndexRename { id: task_id }, - BatchStopReason::TaskCannotBeBatched { kind, id: task_id }, - )), - false, - ), K::IndexSwap => ( Break(( BatchKind::IndexSwap { id: task_id }, @@ -299,8 +287,8 @@ impl BatchKind { }; match (self, autobatch_kind) { - // We don't batch any of these operations - (this, K::IndexCreation | K::IndexUpdate | K::IndexRename | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), + // We don't batch any of these operations + (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists. (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => { Break((this, BatchStopReason::IndexCreationMismatch { id })) diff --git a/crates/index-scheduler/src/scheduler/autobatcher_test.rs b/crates/index-scheduler/src/scheduler/autobatcher_test.rs index 435ce55c6..0653753dc 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher_test.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher_test.rs @@ -75,7 +75,11 @@ fn idx_create() -> KindWithContent { } fn idx_update() -> KindWithContent { - KindWithContent::IndexUpdate { index_uid: String::from("doggo"), primary_key: None } + KindWithContent::IndexUpdate { + index_uid: String::from("doggo"), + primary_key: None, + new_index_uid: None, + } } fn idx_del() -> KindWithContent { diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index fa84200a1..14aa307cd 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -38,11 +38,7 @@ pub(crate) enum Batch { IndexUpdate { index_uid: String, primary_key: Option, - task: Task, - }, - IndexRename { - index_uid: String, - new_index_uid: String, + new_index_uid: Option, task: Task, }, IndexDeletion { @@ -113,8 +109,7 @@ impl Batch { | Batch::Dump(task) | Batch::IndexCreation { task, .. } | Batch::Export { task } - | Batch::IndexUpdate { task, .. } - | Batch::IndexRename { task, .. } => { + | Batch::IndexUpdate { task, .. } => { RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap() } Batch::SnapshotCreation(tasks) @@ -159,7 +154,6 @@ impl Batch { IndexOperation { op, .. } => Some(op.index_uid()), IndexCreation { index_uid, .. } | IndexUpdate { index_uid, .. } - | IndexRename { index_uid, .. } | IndexDeletion { index_uid, .. } => Some(index_uid), } } @@ -178,7 +172,6 @@ impl fmt::Display for Batch { Batch::IndexOperation { op, .. } => write!(f, "{op}")?, Batch::IndexCreation { .. } => f.write_str("IndexCreation")?, Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?, - Batch::IndexRename { .. } => f.write_str("IndexRename")?, Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?, Batch::IndexSwap { .. } => f.write_str("IndexSwap")?, Batch::Export { .. } => f.write_str("Export")?, @@ -413,21 +406,13 @@ impl IndexScheduler { let mut task = self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; current_batch.processing(Some(&mut task)); - let primary_key = match &task.kind { - KindWithContent::IndexUpdate { primary_key, .. } => primary_key.clone(), + let (primary_key, new_index_uid) = match &task.kind { + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + (primary_key.clone(), new_index_uid.clone()) + } _ => unreachable!(), }; - Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task })) - } - BatchKind::IndexRename { id } => { - let mut task = - self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; - current_batch.processing(Some(&mut task)); - let (new_uid) = match &task.kind { - KindWithContent::IndexRename { new_index_uid, .. } => new_index_uid.clone(), - _ => unreachable!(), - }; - Ok(Some(Batch::IndexRename { index_uid, new_index_uid: new_uid, task })) + Ok(Some(Batch::IndexUpdate { index_uid, primary_key, new_index_uid, task })) } BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion { index_uid, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index d801287a7..a82324fc1 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -15,7 +15,7 @@ use super::create_batch::Batch; use crate::processing::{ AtomicBatchStep, AtomicTaskStep, CreateIndexProgress, DeleteIndexProgress, FinalizingIndexStep, InnerSwappingTwoIndexes, SwappingTheIndexes, TaskCancelationProgress, TaskDeletionProgress, - UpdateIndexProgress, RenameIndexProgress, + UpdateIndexProgress, }; use crate::utils::{ self, remove_n_tasks_datetime_earlier_than, remove_task_datetime, swap_index_uid_in_task, @@ -224,38 +224,47 @@ impl IndexScheduler { self.index_mapper.create_index(wtxn, &index_uid, None)?; self.process_batch( - Batch::IndexUpdate { index_uid, primary_key, task }, + Batch::IndexUpdate { index_uid, primary_key, new_index_uid: None, task }, current_batch, progress, ) } - Batch::IndexRename { index_uid, new_index_uid, mut task } => { - progress.update_progress(RenameIndexProgress::RenamingTheIndex); - let mut wtxn = self.env.write_txn()?; - self.index_mapper.rename(&mut wtxn, &index_uid, &new_index_uid)?; - self.queue.tasks.update_index(&mut wtxn, &new_index_uid, |bm| { - let old = self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); - *bm |= &old; - })?; - self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; - wtxn.commit()?; - task.status = Status::Succeeded; - task.details = Some(Details::IndexRename(IndexRenameDetails { old_uid: index_uid, new_uid: new_index_uid })); - Ok((vec![task], ProcessBatchInfo::default())) - } - Batch::IndexUpdate { index_uid, primary_key, mut task } => { + Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); - let rtxn = self.env.read_txn()?; - let index = self.index_mapper.index(&rtxn, &index_uid)?; - if let Some(primary_key) = primary_key.clone() { + // Handle rename if new_index_uid is provided + let final_index_uid = if let Some(new_uid) = &new_index_uid { + let mut wtxn = self.env.write_txn()?; + + // Rename the index + self.index_mapper.rename(&mut wtxn, &index_uid, new_uid)?; + + // Update the task index mappings + let old_tasks = + self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); + self.queue.tasks.update_index(&mut wtxn, new_uid, |bm| { + *bm |= &old_tasks; + })?; + self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; + wtxn.commit()?; + + new_uid.clone() + } else { + index_uid.clone() + }; + // Get the index (renamed or not) + let rtxn = self.env.read_txn()?; + let index = self.index_mapper.index(&rtxn, &final_index_uid)?; + + // Handle primary key update if provided + if let Some(ref primary_key) = primary_key { let mut index_wtxn = index.write_txn()?; let mut builder = MilliSettings::new( &mut index_wtxn, &index, self.index_mapper.indexer_config(), ); - builder.set_primary_key(primary_key); + builder.set_primary_key(primary_key.clone()); let must_stop_processing = self.scheduler.must_stop_processing.clone(); builder @@ -264,7 +273,7 @@ impl IndexScheduler { &progress, current_batch.embedder_stats.clone(), ) - .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?; index_wtxn.commit()?; } @@ -272,7 +281,10 @@ impl IndexScheduler { rtxn.commit()?; task.status = Status::Succeeded; - task.details = Some(Details::IndexInfo { primary_key }); + task.details = Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_uid: new_index_uid.clone(), + }); // if the update processed successfully, we're going to store the new // stats of the index. Since the tasks have already been processed and @@ -282,8 +294,8 @@ impl IndexScheduler { let mut wtxn = self.env.write_txn()?; let index_rtxn = index.read_txn()?; let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn) - .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; - self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.clone())))?; + self.index_mapper.store_stats_of(&mut wtxn, &final_index_uid, &stats)?; wtxn.commit()?; Ok(()) }(); diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 3c921f099..91bba35d7 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -264,7 +264,12 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid), K::IndexDeletion { index_uid } => index_uids.push(index_uid), K::IndexCreation { index_uid, .. } => index_uids.push(index_uid), - K::IndexUpdate { index_uid, .. } => index_uids.push(index_uid), + K::IndexUpdate { index_uid, new_index_uid, .. } => { + index_uids.push(index_uid); + if let Some(new_uid) = new_index_uid { + index_uids.push(new_uid); + } + } K::IndexSwap { swaps } => { for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { @@ -496,9 +501,9 @@ impl crate::IndexScheduler { Details::SettingsUpdate { settings: _ } => { assert_eq!(kind.as_kind(), Kind::SettingsUpdate); } - Details::IndexInfo { primary_key: pk1 } => match &kind { + Details::IndexInfo { primary_key: pk1, .. } => match &kind { KindWithContent::IndexCreation { index_uid, primary_key: pk2 } - | KindWithContent::IndexUpdate { index_uid, primary_key: pk2 } => { + | KindWithContent::IndexUpdate { index_uid, primary_key: pk2, .. } => { self.queue .tasks .index_tasks diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 7521137c0..460ae68d7 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -132,6 +132,11 @@ pub struct DetailsView { pub payload_size: Option, #[serde(skip_serializing_if = "Option::is_none")] pub indexes: Option>, + // index rename + #[serde(skip_serializing_if = "Option::is_none")] + pub old_index_uid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_index_uid: Option, } impl DetailsView { @@ -292,6 +297,18 @@ impl DetailsView { (None, Some(to)) | (Some(to), None) => Some(to), (Some(_), Some(to)) => Some(to), }, + old_index_uid: match (self.old_index_uid.clone(), other.old_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, + new_index_uid: match (self.new_index_uid.clone(), other.new_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, } } } @@ -324,9 +341,11 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key } => { - DetailsView { primary_key: Some(primary_key), ..DetailsView::default() } - } + Details::IndexInfo { primary_key, new_uid } => DetailsView { + primary_key: Some(primary_key), + new_index_uid: new_uid.clone(), + ..DetailsView::default() + }, Details::DocumentDeletion { provided_ids: received_document_ids, deleted_documents, diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 64826c693..f7b59b299 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -140,10 +140,7 @@ pub enum KindWithContent { IndexUpdate { index_uid: String, primary_key: Option, - }, - IndexRename { - index_uid: String, - new_index_uid: String, + new_index_uid: Option, }, IndexSwap { swaps: Vec, @@ -178,13 +175,6 @@ pub struct IndexSwap { pub indexes: (String, String), } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct IndexRenameDetails { - pub old_uid: String, - pub new_uid: String, -} - #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct ExportIndexSettings { @@ -204,7 +194,6 @@ impl KindWithContent { KindWithContent::IndexCreation { .. } => Kind::IndexCreation, KindWithContent::IndexDeletion { .. } => Kind::IndexDeletion, KindWithContent::IndexUpdate { .. } => Kind::IndexUpdate, - KindWithContent::IndexRename { .. } => Kind::IndexRename, KindWithContent::IndexSwap { .. } => Kind::IndexSwap, KindWithContent::TaskCancelation { .. } => Kind::TaskCancelation, KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion, @@ -232,9 +221,14 @@ impl KindWithContent { | DocumentClear { index_uid } | SettingsUpdate { index_uid, .. } | IndexCreation { index_uid, .. } - | IndexUpdate { index_uid, .. } | IndexDeletion { index_uid } => vec![index_uid], - IndexRename { index_uid, new_index_uid } => vec![index_uid, new_index_uid], + IndexUpdate { index_uid, new_index_uid, .. } => { + let mut indexes = vec![index_uid.as_str()]; + if let Some(new_uid) = new_index_uid { + indexes.push(new_uid.as_str()); + } + indexes + } IndexSwap { swaps } => { let mut indexes = HashSet::<&str>::default(); for swap in swaps { @@ -283,13 +277,12 @@ impl KindWithContent { KindWithContent::SettingsUpdate { new_settings, .. } => { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexCreation { primary_key, .. } => { + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) } - KindWithContent::IndexRename { index_uid, new_index_uid } => { - Some(Details::IndexRename { - old_uid: index_uid.clone(), + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), new_uid: new_index_uid.clone(), }) } @@ -363,16 +356,15 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexRename { index_uid, new_index_uid } => { - Some(Details::IndexRename { - old_uid: index_uid.clone(), + KindWithContent::IndexCreation { primary_key, .. } => { + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + } + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), new_uid: new_index_uid.clone(), }) } - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) - } KindWithContent::IndexSwap { .. } => { todo!() } @@ -426,10 +418,13 @@ impl From<&KindWithContent> for Option
{ } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) } - KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_uid: new_index_uid.clone(), + }) } KindWithContent::IndexSwap { .. } => None, KindWithContent::TaskCancelation { query, tasks } => Some(Details::TaskCancelation { @@ -563,7 +558,6 @@ pub enum Kind { IndexCreation, IndexDeletion, IndexUpdate, - IndexRename, IndexSwap, TaskCancelation, TaskDeletion, @@ -582,8 +576,7 @@ impl Kind { | Kind::SettingsUpdate | Kind::IndexCreation | Kind::IndexDeletion - | Kind::IndexUpdate - | Kind::IndexRename => true, + | Kind::IndexUpdate => true, Kind::IndexSwap | Kind::TaskCancelation | Kind::TaskDeletion @@ -604,7 +597,6 @@ impl Display for Kind { Kind::IndexCreation => write!(f, "indexCreation"), Kind::IndexDeletion => write!(f, "indexDeletion"), Kind::IndexUpdate => write!(f, "indexUpdate"), - Kind::IndexRename => write!(f, "indexRename"), Kind::IndexSwap => write!(f, "indexSwap"), Kind::TaskCancelation => write!(f, "taskCancelation"), Kind::TaskDeletion => write!(f, "taskDeletion"), @@ -623,8 +615,6 @@ impl FromStr for Kind { Ok(Kind::IndexCreation) } else if kind.eq_ignore_ascii_case("indexUpdate") { Ok(Kind::IndexUpdate) - } else if kind.eq_ignore_ascii_case("indexRename") { - Ok(Kind::IndexRename) } else if kind.eq_ignore_ascii_case("indexSwap") { Ok(Kind::IndexSwap) } else if kind.eq_ignore_ascii_case("indexDeletion") { @@ -687,6 +677,7 @@ pub enum Details { }, IndexInfo { primary_key: Option, + new_uid: Option, }, DocumentDeletion { provided_ids: usize, @@ -722,7 +713,6 @@ pub enum Details { IndexSwap { swaps: Vec, }, - IndexRename(IndexRenameDetails), Export { url: String, api_key: Option, @@ -768,7 +758,6 @@ impl Details { Self::SettingsUpdate { .. } | Self::IndexInfo { .. } | Self::Dump { .. } - | Self::IndexRename { .. } | Self::Export { .. } | Self::UpgradeDatabase { .. } | Self::IndexSwap { .. } => (), diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 04b3e12c4..632922542 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -375,6 +375,9 @@ pub struct UpdateIndexRequest { /// The new primary key of the index #[deserr(default, error = DeserrJsonError)] primary_key: Option, + /// The new uid of the index (for renaming) + #[deserr(default, error = DeserrJsonError)] + uid: Option, } /// Update index @@ -419,6 +422,12 @@ pub async fn update_index( debug!(parameters = ?body, "Update index"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let body = body.into_inner(); + + // Validate new uid if provided + if let Some(ref new_uid) = body.uid { + let _ = IndexUid::try_from(new_uid.clone())?; + } + analytics.publish( IndexUpdatedAggregate { primary_key: body.primary_key.iter().cloned().collect() }, &req, @@ -427,6 +436,7 @@ pub async fn update_index( let task = KindWithContent::IndexUpdate { index_uid: index_uid.into_inner(), primary_key: body.primary_key, + new_index_uid: body.uid, }; let uid = get_task_id(&req, &opt)?; diff --git a/crates/meilisearch/tests/index/mod.rs b/crates/meilisearch/tests/index/mod.rs index 5df5e7e97..f3cb7fde9 100644 --- a/crates/meilisearch/tests/index/mod.rs +++ b/crates/meilisearch/tests/index/mod.rs @@ -2,5 +2,6 @@ mod create_index; mod delete_index; mod errors; mod get_index; +mod rename_index; mod stats; mod update_index; diff --git a/crates/meilisearch/tests/index/rename_index.rs b/crates/meilisearch/tests/index/rename_index.rs new file mode 100644 index 000000000..793b079f1 --- /dev/null +++ b/crates/meilisearch/tests/index/rename_index.rs @@ -0,0 +1,419 @@ +use crate::common::{shared_does_not_exists_index, Server}; +use crate::json; + +#[actix_rt::test] +async fn rename_index_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename via PATCH update endpoint + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); +} + +#[actix_rt::test] +async fn rename_to_existing_index_via_patch() { + let server = Server::new_shared(); + let index1 = server.unique_index(); + let index2 = server.unique_index(); + + // Create both indexes + let (task, code) = index1.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let (task, code) = index2.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename index1 to index2's uid via PATCH (should fail) + let body = json!({ "uid": index2.uid }); + let (task, code) = index1.service.patch(format!("/indexes/{}", index1.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` already exists.", index2.uid), + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_non_existent_index_via_patch() { + let server = Server::new_shared(); + let index = shared_does_not_exists_index().await; + + // Try to rename non-existent index via PATCH + let body = json!({ "uid": "new_name" }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` not found.", index.uid), + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_with_invalid_uid_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename with invalid uid via PATCH + let body = json!({ "uid": "Invalid UID!" }); + let (_, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn rename_index_with_documents_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1" }, + { "id": 2, "title": "Movie 2" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index via PATCH + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify documents are accessible in renamed index + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn rename_index_and_update_primary_key_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index without primary key + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename index and set primary key at the same time + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ + "uid": &new_uid, + "primaryKey": "id" + }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded and primary key was set + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + assert_eq!(response["details"]["primaryKey"], "id"); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists with correct primary key + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); + assert_eq!(response["primaryKey"], "id"); +} + +#[actix_rt::test] +async fn rename_index_and_verify_stats() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1", "genre": "Action" }, + { "id": 2, "title": "Movie 2", "genre": "Drama" }, + { "id": 3, "title": "Movie 3", "genre": "Comedy" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats before rename + let (stats_before, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!(stats_before["numberOfDocuments"], 3); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats after rename using the new uid + let (stats_after, code) = server.service.get(format!("/indexes/{}/stats", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(stats_after["numberOfDocuments"], 3); + assert_eq!(stats_after["numberOfDocuments"], stats_before["numberOfDocuments"]); +} + +#[actix_rt::test] +async fn rename_index_preserves_settings() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Configure settings + let settings = json!({ + "searchableAttributes": ["title", "description"], + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": ["the", "a", "an"], + "synonyms": { + "movie": ["film", "picture"], + "great": ["awesome", "excellent"] + } + }); + + let (task, code) = index.update_settings(settings.clone()).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify settings are preserved + let (settings_after, code) = server.service.get(format!("/indexes/{}/settings", new_uid)).await; + assert_eq!(code, 200); + + assert_eq!(settings_after["searchableAttributes"], json!(["title", "description"])); + assert_eq!(settings_after["filterableAttributes"], json!(["genre", "year"])); + assert_eq!(settings_after["sortableAttributes"], json!(["year"])); + + // Check stopWords contains the same items (order may vary) + let stop_words = settings_after["stopWords"].as_array().unwrap(); + assert_eq!(stop_words.len(), 3); + assert!(stop_words.contains(&json!("the"))); + assert!(stop_words.contains(&json!("a"))); + assert!(stop_words.contains(&json!("an"))); + + assert_eq!(settings_after["synonyms"]["movie"], json!(["film", "picture"])); + assert_eq!(settings_after["synonyms"]["great"], json!(["awesome", "excellent"])); +} + +#[actix_rt::test] +async fn rename_index_preserves_search_functionality() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "The Matrix", "genre": "Sci-Fi", "year": 1999 }, + { "id": 2, "title": "Inception", "genre": "Sci-Fi", "year": 2010 }, + { "id": 3, "title": "The Dark Knight", "genre": "Action", "year": 2008 } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Make settings filterable + let settings = json!({ + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"] + }); + let (task, code) = index.update_settings(settings).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search before rename + let search_params = json!({ + "q": "matrix", + "filter": "genre = 'Sci-Fi'", + "sort": ["year:asc"] + }); + let (results_before, code) = index.search_post(search_params.clone()).await; + assert_eq!(code, 200); + assert_eq!(results_before["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_before["hits"][0]["title"], "The Matrix"); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search after rename + let (results_after, code) = + server.service.post(format!("/indexes/{}/search", new_uid), search_params).await; + assert_eq!(code, 200); + assert_eq!(results_after["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_after["hits"][0]["title"], "The Matrix"); + + // Verify facet search also works + let facet_search = json!({ + "facetQuery": "Sci", + "facetName": "genre" + }); + let (facet_results, code) = + server.service.post(format!("/indexes/{}/facet-search", new_uid), facet_search).await; + assert_eq!(code, 200); + assert_eq!(facet_results["facetHits"].as_array().unwrap().len(), 1); + assert_eq!(facet_results["facetHits"][0]["value"], "Sci-Fi"); +} + +#[actix_rt::test] +async fn rename_index_with_pending_tasks() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Add initial documents + let documents = json!([ + { "id": 1, "title": "Document 1" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Start a rename + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (rename_task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + assert_eq!(code, 202); + + // Try to add documents to the old index while rename is pending + let more_documents = json!([ + { "id": 2, "title": "Document 2" } + ]); + let (_, code) = index.add_documents(more_documents, None).await; + assert_eq!(code, 202); + + // Wait for rename to complete + server.wait_task(rename_task.uid()).await.succeeded(); + + // Add documents to the new index + let final_documents = json!([ + { "id": 3, "title": "Document 3" } + ]); + let (task, code) = + server.service.post(format!("/indexes/{}/documents", new_uid), final_documents).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify all documents are accessible + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + let docs = response["results"].as_array().unwrap(); + assert!(!docs.is_empty()); // At least the initial document should be there +} + +#[actix_rt::test] +async fn rename_index_to_same_name() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename to the same name + let body = json!({ "uid": index.uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + // Should fail with index already exists error + assert_eq!(response["status"], "failed"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["error"]["code"], "index_already_exists"); + + // Index should still be accessible with original name + let (_, code) = index.get().await; + assert_eq!(code, 200); +} From ae5bd9d0e338cceb121b3975fb0899e5d50bf102 Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Aug 2025 17:34:11 +0200 Subject: [PATCH 03/38] fix: updated_at was not 'updated' when updating the index name --- .../src/scheduler/process_batch.rs | 14 +++++--- .../meilisearch/tests/index/update_index.rs | 33 +++++++++++++++++++ crates/milli/src/index.rs | 2 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index a82324fc1..beb7c021f 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -10,6 +10,7 @@ use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use milli::update::Settings as MilliSettings; use roaring::RoaringBitmap; +use time::OffsetDateTime; use super::create_batch::Batch; use crate::processing::{ @@ -232,8 +233,15 @@ impl IndexScheduler { Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); + // Get the index (renamed or not) + let rtxn = self.env.read_txn()?; + let index = self.index_mapper.index(&rtxn, &index_uid)?; + let mut index_wtxn = index.write_txn()?; + // Handle rename if new_index_uid is provided let final_index_uid = if let Some(new_uid) = &new_index_uid { + index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; + let mut wtxn = self.env.write_txn()?; // Rename the index @@ -252,13 +260,9 @@ impl IndexScheduler { } else { index_uid.clone() }; - // Get the index (renamed or not) - let rtxn = self.env.read_txn()?; - let index = self.index_mapper.index(&rtxn, &final_index_uid)?; // Handle primary key update if provided if let Some(ref primary_key) = primary_key { - let mut index_wtxn = index.write_txn()?; let mut builder = MilliSettings::new( &mut index_wtxn, &index, @@ -274,9 +278,9 @@ impl IndexScheduler { current_batch.embedder_stats.clone(), ) .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?; - index_wtxn.commit()?; } + index_wtxn.commit()?; // drop rtxn before starting a new wtxn on the same db rtxn.commit()?; diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 262324bcf..45f3ea420 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -1,3 +1,5 @@ +use meili_snap::snapshot; + use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; @@ -106,3 +108,34 @@ async fn error_update_unexisting_index() { assert_eq!(response["error"], expected_response); } + +#[actix_rt::test] +async fn update_index_name() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, code) = index.create(None).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let new_index = server.unique_index(); + let (task, _status_code) = index.update_raw(json!({ "uid": new_index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (response, code) = new_index.get().await; + + snapshot!(code, @"200 OK"); + + assert_eq!(response["uid"], new_index.uid); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + + let created_at = + OffsetDateTime::parse(response["createdAt"].as_str().unwrap(), &Rfc3339).unwrap(); + let updated_at = + OffsetDateTime::parse(response["updatedAt"].as_str().unwrap(), &Rfc3339).unwrap(); + assert!(created_at < updated_at, "{created_at} should be inferior to {updated_at}"); + + snapshot!(response["primaryKey"], @"null"); + snapshot!(response.as_object().unwrap().len(), @"4"); +} diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 9f32fdb04..6429dabbc 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1457,7 +1457,7 @@ impl Index { .0) } - pub(crate) fn set_updated_at( + pub fn set_updated_at( &self, wtxn: &mut RwTxn<'_>, time: &time::OffsetDateTime, From ecea247e5d3e30c2562e3714114a44f726e6d4bf Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 7 Aug 2025 19:35:07 +0200 Subject: [PATCH 04/38] Provide a rename argument to the swap --- crates/index-scheduler/src/error.rs | 10 + .../index-scheduler/src/queue/batches_test.rs | 6 +- .../index-scheduler/src/queue/tasks_test.rs | 6 +- .../src/scheduler/autobatcher_test.rs | 5 +- .../src/scheduler/process_batch.rs | 53 +++- crates/index-scheduler/src/scheduler/test.rs | 16 +- crates/index-scheduler/src/utils.rs | 6 +- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch-types/src/tasks.rs | 1 + crates/meilisearch/src/routes/swap_indexes.rs | 19 +- .../meilisearch/tests/swap_indexes/errors.rs | 17 ++ crates/meilisearch/tests/swap_indexes/mod.rs | 252 ++++++++++++++++++ 12 files changed, 358 insertions(+), 34 deletions(-) diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 60669ff2d..742576c73 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -67,6 +67,8 @@ pub enum Error { SwapDuplicateIndexesFound(Vec), #[error("Index `{0}` not found.")] SwapIndexNotFound(String), + #[error("Index `{0}` found during a rename. Renaming doen't overwrite the other index name.")] + SwapIndexFoundDuringRename(String), #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")] NoSpaceLeftInTaskQueue, #[error( @@ -74,6 +76,10 @@ pub enum Error { .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesNotFound(Vec), + #[error("Index {} found during a rename. Renaming doen't overwrite the other index name.", + .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") + )] + SwapIndexesFoundDuringRename(Vec), #[error("Corrupted dump.")] CorruptedDump, #[error( @@ -203,6 +209,8 @@ impl Error { | Error::SwapIndexNotFound(_) | Error::NoSpaceLeftInTaskQueue | Error::SwapIndexesNotFound(_) + | Error::SwapIndexFoundDuringRename(_) + | Error::SwapIndexesFoundDuringRename(_) | Error::CorruptedDump | Error::InvalidTaskDate { .. } | Error::InvalidTaskUid { .. } @@ -271,6 +279,8 @@ impl ErrorCode for Error { Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound, Error::SwapIndexNotFound(_) => Code::IndexNotFound, Error::SwapIndexesNotFound(_) => Code::IndexNotFound, + Error::SwapIndexFoundDuringRename(_) => Code::IndexNotFound, + Error::SwapIndexesFoundDuringRename(_) => Code::IndexNotFound, Error::InvalidTaskDate { field, .. } => (*field).into(), Error::InvalidTaskUid { .. } => Code::InvalidTaskUids, Error::InvalidBatchUid { .. } => Code::InvalidBatchUids, diff --git a/crates/index-scheduler/src/queue/batches_test.rs b/crates/index-scheduler/src/queue/batches_test.rs index 782acb4b1..49dc0ba61 100644 --- a/crates/index-scheduler/src/queue/batches_test.rs +++ b/crates/index-scheduler/src/queue/batches_test.rs @@ -334,11 +334,11 @@ fn query_batches_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -442,7 +442,7 @@ fn query_batches_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/queue/tasks_test.rs b/crates/index-scheduler/src/queue/tasks_test.rs index d60d621d1..bfe26accf 100644 --- a/crates/index-scheduler/src/queue/tasks_test.rs +++ b/crates/index-scheduler/src/queue/tasks_test.rs @@ -304,11 +304,11 @@ fn query_tasks_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -399,7 +399,7 @@ fn query_tasks_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/scheduler/autobatcher_test.rs b/crates/index-scheduler/src/scheduler/autobatcher_test.rs index 0653753dc..7dd4536b7 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher_test.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher_test.rs @@ -88,7 +88,10 @@ fn idx_del() -> KindWithContent { fn idx_swap() -> KindWithContent { KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: (String::from("doggo"), String::from("catto")) }], + swaps: vec![IndexSwap { + indexes: (String::from("doggo"), String::from("catto")), + rename: false, + }], } } diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index beb7c021f..7688b1787 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -360,13 +360,18 @@ impl IndexScheduler { unreachable!() }; let mut not_found_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { - for index in [lhs, rhs] { - let index_exists = self.index_mapper.index_exists(&wtxn, index)?; - if !index_exists { - not_found_indexes.insert(index); - } + let mut found_indexes_but_should_not = BTreeSet::new(); + for IndexSwap { indexes: (lhs, rhs), rename } in swaps { + let index_exists = self.index_mapper.index_exists(&wtxn, lhs)?; + if !index_exists { + not_found_indexes.insert(lhs); } + let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?; + match (index_exists, rename) { + (true, true) => found_indexes_but_should_not.insert(rhs), + (false, false) => not_found_indexes.insert(rhs), + (true, false) | (false, true) => true, // random value we don't read it anyway + }; } if !not_found_indexes.is_empty() { if not_found_indexes.len() == 1 { @@ -379,6 +384,17 @@ impl IndexScheduler { )); } } + if !found_indexes_but_should_not.is_empty() { + if found_indexes_but_should_not.len() == 1 { + return Err(Error::SwapIndexFoundDuringRename( + found_indexes_but_should_not.into_iter().next().unwrap().clone(), + )); + } else { + return Err(Error::SwapIndexesFoundDuringRename( + found_indexes_but_should_not.into_iter().cloned().collect(), + )); + } + } progress.update_progress(SwappingTheIndexes::SwappingTheIndexes); for (step, swap) in swaps.iter().enumerate() { progress.update_progress(VariableNameStep::::new( @@ -392,6 +408,7 @@ impl IndexScheduler { task.uid, &swap.indexes.0, &swap.indexes.1, + swap.rename, )?; } wtxn.commit()?; @@ -481,6 +498,7 @@ impl IndexScheduler { task_id: u32, lhs: &str, rhs: &str, + rename: bool, ) -> Result<()> { progress.update_progress(InnerSwappingTwoIndexes::RetrieveTheTasks); // 1. Verify that both lhs and rhs are existing indexes @@ -488,16 +506,23 @@ impl IndexScheduler { if !index_lhs_exists { return Err(Error::IndexNotFound(lhs.to_owned())); } - let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; - if !index_rhs_exists { - return Err(Error::IndexNotFound(rhs.to_owned())); + if !rename { + let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; + if !index_rhs_exists { + return Err(Error::IndexNotFound(rhs.to_owned())); + } } // 2. Get the task set for index = name that appeared before the index swap task let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?; index_lhs_task_ids.remove_range(task_id..); - let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; - index_rhs_task_ids.remove_range(task_id..); + let index_rhs_task_ids = if rename { + let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; + index_rhs_task_ids.remove_range(task_id..); + index_rhs_task_ids + } else { + RoaringBitmap::new() + }; // 3. before_name -> new_name in the task's KindWithContent progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks); @@ -526,7 +551,11 @@ impl IndexScheduler { })?; // 6. Swap in the index mapper - self.index_mapper.swap(wtxn, lhs, rhs)?; + if rename { + self.index_mapper.rename(wtxn, lhs, rhs)?; + } else { + self.index_mapper.swap(wtxn, lhs, rhs)?; + } Ok(()) } diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index e9f21dfe4..8cc1b8830 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -372,8 +372,8 @@ fn swap_indexes() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "d".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "d".to_owned()), rename: false }, ], }, None, @@ -384,7 +384,7 @@ fn swap_indexes() { index_scheduler .register( KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()), rename: false }], }, None, false, @@ -428,8 +428,8 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("b".to_owned(), "a".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("b".to_owned(), "a".to_owned()), rename: false }, ], }, None, @@ -446,9 +446,9 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "e".to_owned()) }, - IndexSwap { indexes: ("d".to_owned(), "f".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "e".to_owned()), rename: false }, + IndexSwap { indexes: ("d".to_owned(), "f".to_owned()), rename: false }, ], }, None, diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 91bba35d7..159e8f3d3 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -271,7 +271,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { } } K::IndexSwap { swaps } => { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -288,7 +288,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { | K::SnapshotCreation => (), }; if let Some(Details::IndexSwap { swaps }) = &mut task.details { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -330,7 +330,7 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { if let KindWithContent::IndexSwap { swaps } = &task.kind { swaps } else { return Ok(()) }; let mut all_indexes = HashSet::new(); let mut duplicate_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps { for name in [lhs, rhs] { let is_new = all_indexes.insert(name); if !is_new { diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 4360d947b..415bb5fdb 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -335,6 +335,7 @@ InvalidState , Internal , INTERNAL InvalidStoreFile , Internal , INTERNAL_SERVER_ERROR ; InvalidSwapDuplicateIndexFound , InvalidRequest , BAD_REQUEST ; InvalidSwapIndexes , InvalidRequest , BAD_REQUEST ; +InvalidSwapRename , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterEnqueuedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterFinishedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterStartedAt , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index f7b59b299..fbdaac9ce 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -173,6 +173,7 @@ pub enum KindWithContent { #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), + pub rename: bool, } #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] diff --git a/crates/meilisearch/src/routes/swap_indexes.rs b/crates/meilisearch/src/routes/swap_indexes.rs index 4a35d1a6d..533d2caf1 100644 --- a/crates/meilisearch/src/routes/swap_indexes.rs +++ b/crates/meilisearch/src/routes/swap_indexes.rs @@ -4,7 +4,7 @@ use deserr::actix_web::AwebJson; use deserr::Deserr; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::InvalidSwapIndexes; +use meilisearch_types::error::deserr_codes::{InvalidSwapIndexes, InvalidSwapRename}; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; @@ -33,11 +33,15 @@ pub struct SwapIndexesPayload { /// Array of the two indexUids to be swapped #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_swap_indexes)] indexes: Vec, + /// If set to true, instead of swapping the left and right indexes it'll change the name of the first index to the second + #[deserr(default, error = DeserrJsonError)] + rename: bool, } #[derive(Serialize)] struct IndexSwappedAnalytics { swap_operation_number: usize, + rename_used: bool, } impl Aggregate for IndexSwappedAnalytics { @@ -48,6 +52,7 @@ impl Aggregate for IndexSwappedAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(Self { swap_operation_number: self.swap_operation_number.max(new.swap_operation_number), + rename_used: self.rename_used | new.rename_used, }) } @@ -95,11 +100,17 @@ pub async fn swap_indexes( analytics: web::Data, ) -> Result { let params = params.into_inner(); - analytics.publish(IndexSwappedAnalytics { swap_operation_number: params.len() }, &req); + analytics.publish( + IndexSwappedAnalytics { + swap_operation_number: params.len(), + rename_used: params.iter().any(|obj| obj.rename), + }, + &req, + ); let filters = index_scheduler.filters(); let mut swaps = vec![]; - for SwapIndexesPayload { indexes } in params.into_iter() { + for SwapIndexesPayload { indexes, rename } in params.into_iter() { // TODO: switch to deserr let (lhs, rhs) = match indexes.as_slice() { [lhs, rhs] => (lhs, rhs), @@ -110,7 +121,7 @@ pub async fn swap_indexes( if !filters.is_index_authorized(lhs) || !filters.is_index_authorized(rhs) { return Err(AuthenticationError::InvalidToken.into()); } - swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()) }); + swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()), rename }); } let task = KindWithContent::IndexSwap { swaps }; diff --git a/crates/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs index d136dfc0f..63dfdfe8a 100644 --- a/crates/meilisearch/tests/swap_indexes/errors.rs +++ b/crates/meilisearch/tests/swap_indexes/errors.rs @@ -92,3 +92,20 @@ async fn swap_indexes_bad_indexes() { } "###); } + +#[actix_rt::test] +async fn swap_indexes_bad_rename() { + let server = Server::new_shared(); + + let (response, code) = + server.index_swap(json!([{ "indexes": ["kefir", "intel"], "rename": "hello" }])).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r#" + { + "message": "Invalid value type at `[0].rename`: expected a boolean, but found a string: `\"hello\"`", + "code": "invalid_swap_rename", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_swap_rename" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs index bf84d5823..780dc2d86 100644 --- a/crates/meilisearch/tests/swap_indexes/mod.rs +++ b/crates/meilisearch/tests/swap_indexes/mod.rs @@ -372,3 +372,255 @@ async fn swap_indexes() { let (res, _) = d.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(res["results"], @r###"[{"id":1,"index":"c"}]"###); } + +#[actix_rt::test] +async fn swap_rename_indexes() { + let server = Server::new().await; + let a = server.index("a"); + let b = server.index("b"); + a.create(None).await; + a.add_documents(json!({ "id": 1, "index": "a"}), None).await; + + let (res, _code) = server.index_swap(json!([{ "indexes": ["a", "b"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + let (tasks, _code) = server.tasks().await; + + // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 2, + "batchUid": 2, + "indexUid": null, + "status": "succeeded", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "a", + "b" + ], + "rename": true + } + ] + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 2, + "next": null + } + "#); + + // BUT, `a` should not exists + let (res, code) = a.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"404 Not Found"); + snapshot!(res["results"], @"null"); + + // And its data should be in b + let (res, code) = b.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"200 OK"); + snapshot!(res["results"], @r#"[{"id":1,"index":"a"}]"#); + + // No tasks should be linked to the index a + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 1, + "limit": 20, + "from": null, + "next": null + } + "#); + + // They should be linked to the index b + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 1, + "next": null + } + "#); + + // ===== Now, we can delete the index `b`, but its tasks will stays + // if we then make a new `b` index and rename it to be called `a` + // the tasks currently available in `b` should not be moved + + let (res, _) = b.delete().await; + server.wait_task(res.uid()).await.succeeded(); + + let (res, _) = b.create(Some("kefir")).await; + let (res, _code) = server.index_swap(json!([{ "indexes": ["b", "a"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + // `a` now contains everything + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 4, + "batchUid": 4, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": "kefir" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 3, + "batchUid": 3, + "indexUid": "a", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "a", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 6, + "limit": 20, + "from": 4, + "next": null + } + "#); + + // And `b` is empty + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 2, + "limit": 20, + "from": null, + "next": null + } + "#); +} From a904ce109af0526dd060d098fc1aa4d1254ddcac Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 17:37:30 +0200 Subject: [PATCH 05/38] fix error code and add a bunch of tests for the swap and index rename --- crates/index-scheduler/src/error.rs | 12 +-- .../src/scheduler/process_batch.rs | 16 ++- crates/meilisearch/tests/batches/mod.rs | 7 +- .../meilisearch/tests/index/update_index.rs | 47 +++++++-- .../meilisearch/tests/swap_indexes/errors.rs | 97 ++++++++++++++++++- crates/meilisearch/tests/swap_indexes/mod.rs | 28 +++--- crates/meilisearch/tests/tasks/mod.rs | 14 +-- 7 files changed, 182 insertions(+), 39 deletions(-) diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 742576c73..332b7e040 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -67,8 +67,8 @@ pub enum Error { SwapDuplicateIndexesFound(Vec), #[error("Index `{0}` not found.")] SwapIndexNotFound(String), - #[error("Index `{0}` found during a rename. Renaming doen't overwrite the other index name.")] - SwapIndexFoundDuringRename(String), + #[error("Cannot rename `{0}` to `{1}` as the index already exists. Hint: You can remove `{1}` first and then do your remove.")] + SwapIndexFoundDuringRename(String, String), #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")] NoSpaceLeftInTaskQueue, #[error( @@ -76,7 +76,7 @@ pub enum Error { .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesNotFound(Vec), - #[error("Index {} found during a rename. Renaming doen't overwrite the other index name.", + #[error("The following indexes are being renamed but cannot because their new name conflicts with an already existing index: {}. Renaming doesn't overwrite the other index name.", .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesFoundDuringRename(Vec), @@ -209,7 +209,7 @@ impl Error { | Error::SwapIndexNotFound(_) | Error::NoSpaceLeftInTaskQueue | Error::SwapIndexesNotFound(_) - | Error::SwapIndexFoundDuringRename(_) + | Error::SwapIndexFoundDuringRename(_, _) | Error::SwapIndexesFoundDuringRename(_) | Error::CorruptedDump | Error::InvalidTaskDate { .. } @@ -279,8 +279,8 @@ impl ErrorCode for Error { Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound, Error::SwapIndexNotFound(_) => Code::IndexNotFound, Error::SwapIndexesNotFound(_) => Code::IndexNotFound, - Error::SwapIndexFoundDuringRename(_) => Code::IndexNotFound, - Error::SwapIndexesFoundDuringRename(_) => Code::IndexNotFound, + Error::SwapIndexFoundDuringRename(_, _) => Code::IndexAlreadyExists, + Error::SwapIndexesFoundDuringRename(_) => Code::IndexAlreadyExists, Error::InvalidTaskDate { field, .. } => (*field).into(), Error::InvalidTaskUid { .. } => Code::InvalidTaskUids, Error::InvalidBatchUid { .. } => Code::InvalidBatchUids, diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 7688b1787..738df7b5e 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -368,7 +368,7 @@ impl IndexScheduler { } let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?; match (index_exists, rename) { - (true, true) => found_indexes_but_should_not.insert(rhs), + (true, true) => found_indexes_but_should_not.insert((lhs, rhs)), (false, false) => not_found_indexes.insert(rhs), (true, false) | (false, true) => true, // random value we don't read it anyway }; @@ -386,12 +386,18 @@ impl IndexScheduler { } if !found_indexes_but_should_not.is_empty() { if found_indexes_but_should_not.len() == 1 { - return Err(Error::SwapIndexFoundDuringRename( - found_indexes_but_should_not.into_iter().next().unwrap().clone(), - )); + let (lhs, rhs) = found_indexes_but_should_not + .into_iter() + .next() + .map(|(lhs, rhs)| (lhs.clone(), rhs.clone())) + .unwrap(); + return Err(Error::SwapIndexFoundDuringRename(lhs, rhs)); } else { return Err(Error::SwapIndexesFoundDuringRename( - found_indexes_but_should_not.into_iter().cloned().collect(), + found_indexes_but_should_not + .into_iter() + .map(|(_, rhs)| rhs.to_string()) + .collect(), )); } } diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 9d6bee7c1..4477d93e8 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1142,7 +1142,7 @@ async fn test_summarized_index_swap() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, - @r###" + @r#" { "uid": "[uid]", "progress": null, @@ -1152,7 +1152,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -1172,7 +1173,7 @@ async fn test_summarized_index_swap() { "finishedAt": "[date]", "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." } - "###); + "#); let doggos_index = server.unique_index(); doggos_index.create(None).await; diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 45f3ea420..8e5837d81 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -4,7 +4,9 @@ use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; use crate::common::encoder::Encoder; -use crate::common::{shared_does_not_exists_index, shared_index_with_documents, Server}; +use crate::common::{ + shared_does_not_exists_index, shared_empty_index, shared_index_with_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -113,17 +115,14 @@ async fn error_update_unexisting_index() { async fn update_index_name() { let server = Server::new_shared(); let index = server.unique_index(); - let (task, code) = index.create(None).await; - - assert_eq!(code, 202); + let (task, _code) = index.create(None).await; server.wait_task(task.uid()).await.succeeded(); let new_index = server.unique_index(); - let (task, _status_code) = index.update_raw(json!({ "uid": new_index.uid })).await; + let (task, _code) = index.update_raw(json!({ "uid": new_index.uid })).await; server.wait_task(task.uid()).await.succeeded(); let (response, code) = new_index.get().await; - snapshot!(code, @"200 OK"); assert_eq!(response["uid"], new_index.uid); @@ -139,3 +138,39 @@ async fn update_index_name() { snapshot!(response["primaryKey"], @"null"); snapshot!(response.as_object().unwrap().len(), @"4"); } + +#[actix_rt::test] +async fn error_update_index_name_to_already_existing_index() { + let server = Server::new_shared(); + let base_index = shared_empty_index().await; + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + + let (task, _status_code) = index.update_raw(json!({ "uid": base_index.uid })).await; + let task = server.wait_task(task.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "failed", + "type": "indexUpdate", + "canceledBy": null, + "details": { + "primaryKey": null, + "newIndexUid": "EMPTY_INDEX" + }, + "error": { + "message": "Index `EMPTY_INDEX` already exists.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs index 63dfdfe8a..ee992372e 100644 --- a/crates/meilisearch/tests/swap_indexes/errors.rs +++ b/crates/meilisearch/tests/swap_indexes/errors.rs @@ -1,6 +1,9 @@ use meili_snap::*; -use crate::common::Server; +use crate::common::{ + shared_empty_index, shared_index_with_documents, shared_index_with_geo_documents, + shared_index_with_nested_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -109,3 +112,95 @@ async fn swap_indexes_bad_rename() { } "#); } + +#[actix_rt::test] +async fn swap_indexes_rename_to_already_existing_index() { + let server = Server::new_shared(); + let already_existing_index = shared_empty_index().await; + let base_index = shared_index_with_documents().await; + + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + } + ] + }, + "error": { + "message": "Cannot rename `SHARED_DOCUMENTS` to `EMPTY_INDEX` as the index already exists. Hint: You can remove `EMPTY_INDEX` first and then do your remove.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let base_index_2 = shared_index_with_geo_documents().await; + let already_existing_index_2 = shared_index_with_nested_documents().await; + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }, { "indexes": [base_index_2.uid, already_existing_index_2.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + }, + { + "indexes": [ + "SHARED_GEO_DOCUMENTS", + "SHARED_NESTED_DOCUMENTS" + ], + "rename": true + } + ] + }, + "error": { + "message": "The following indexes are being renamed but cannot because their new name conflicts with an already existing index: `EMPTY_INDEX`, `SHARED_NESTED_DOCUMENTS`. Renaming doesn't overwrite the other index name.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs index 780dc2d86..edb7fab49 100644 --- a/crates/meilisearch/tests/swap_indexes/mod.rs +++ b/crates/meilisearch/tests/swap_indexes/mod.rs @@ -73,7 +73,7 @@ async fn swap_indexes() { snapshot!(code, @"200 OK"); // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -89,7 +89,8 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false } ] }, @@ -102,7 +103,7 @@ async fn swap_indexes() { { "uid": 1, "batchUid": 1, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -139,7 +140,7 @@ async fn swap_indexes() { "from": 2, "next": null } - "###); + "#); // BUT, the data in `a` should now points to the data that was in `b`. // And the opposite is true as well @@ -228,7 +229,7 @@ async fn swap_indexes() { // 2. stays unchanged // 3. now have the indexUid `d` instead of `c` // 4. now have the indexUid `c` instead of `d` - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -244,13 +245,15 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false }, { "indexes": [ "c", "d" - ] + ], + "rename": false } ] }, @@ -263,7 +266,7 @@ async fn swap_indexes() { { "uid": 4, "batchUid": 4, - "indexUid": "c", + "indexUid": "d", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -307,7 +310,8 @@ async fn swap_indexes() { "indexes": [ "b", "a" - ] + ], + "rename": false } ] }, @@ -337,7 +341,7 @@ async fn swap_indexes() { { "uid": 0, "batchUid": 0, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -357,7 +361,7 @@ async fn swap_indexes() { "from": 5, "next": null } - "###); + "#); // - The data in `a` should point to `a`. // - The data in `b` should point to `b`. @@ -530,7 +534,7 @@ async fn swap_rename_indexes() { let (res, _) = b.delete().await; server.wait_task(res.uid()).await.succeeded(); - let (res, _) = b.create(Some("kefir")).await; + b.create(Some("kefir")).await; let (res, _code) = server.index_swap(json!([{ "indexes": ["b", "a"], "rename": true }])).await; server.wait_task(res.uid()).await.succeeded(); diff --git a/crates/meilisearch/tests/tasks/mod.rs b/crates/meilisearch/tests/tasks/mod.rs index 09700d3c5..6397ad6ad 100644 --- a/crates/meilisearch/tests/tasks/mod.rs +++ b/crates/meilisearch/tests/tasks/mod.rs @@ -895,7 +895,7 @@ async fn test_summarized_index_swap() { server.wait_task(task.uid()).await.failed(); let (task, _) = server.get_task(task.uid()).await; snapshot!(task, - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -909,7 +909,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -924,7 +925,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); let doggos_index = server.unique_index(); let (task, _code) = doggos_index.create(None).await; @@ -941,7 +942,7 @@ async fn test_summarized_index_swap() { let (task, _) = server.get_task(task.uid()).await; snapshot!(json_string!(task, { ".uid" => "[uid]", ".batchUid" => "[batch_uid]", ".**.indexes[0]" => "doggos", ".**.indexes[1]" => "cattos", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }), - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -955,7 +956,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -965,7 +967,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); } #[actix_web::test] From 4068c58417bf72aafb9bbdb318f47fd0882fe67e Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 18:28:00 +0200 Subject: [PATCH 06/38] change the details of the tasks --- crates/dump/src/lib.rs | 6 ++-- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +-- crates/index-scheduler/src/dump.rs | 4 +-- crates/index-scheduler/src/insta_snapshot.rs | 4 +-- .../query_batches_canceled_by/start.snap | 8 ++--- .../processed_all_tasks.snap | 6 ++-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 7 ++-- .../after-advancing-a-bit.snap | 6 ++-- .../query_batches_simple/end.snap | 6 ++-- .../query_batches_simple/start.snap | 7 ++-- .../after-processing-everything.snap | 12 +++---- .../query_batches_special_rules/start.snap | 9 +++-- .../query_tasks_canceled_by/start.snap | 8 ++--- .../processed_all_tasks.snap | 6 ++-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 7 ++-- .../tasks_test.rs/query_tasks_simple/end.snap | 6 ++-- .../query_tasks_simple/start.snap | 7 ++-- .../query_tasks_special_rules/start.snap | 9 +++-- ...everything_is_successfully_registered.snap | 3 +- .../after_the_second_task_deletion.snap | 4 +-- .../task_deletion_have_been_enqueued.snap | 13 ++++--- .../task_deletion_have_been_processed.snap | 7 ++-- .../task_queue_is_full.snap | 13 ++++--- .../task_deletion_have_not_been_enqueued.snap | 13 ++++--- .../task_queue_is_full.snap | 13 ++++--- .../src/scheduler/process_batch.rs | 28 +++++++-------- .../all_tasks_processed.snap | 6 ++-- .../before_index_creation.snap | 2 +- .../both_task_succeeded.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 3 +- .../registered_the_third_task.snap | 3 +- .../after_batch_creation.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 4 +-- .../registered_the_third_task.snap | 4 +-- .../processed_the_first_task.snap | 4 +-- .../processed_the_second_task.snap | 4 +-- .../processed_the_third_task.snap | 4 +-- .../registered_the_first_task.snap | 3 +- .../registered_the_second_task.snap | 5 ++- .../registered_the_third_task.snap | 5 ++- .../first.snap | 2 +- .../fourth.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../registered_the_fourth_task.snap | 3 +- .../registered_the_second_task.snap | 3 +- .../registered_the_third_task.snap | 3 +- .../second.snap | 2 +- .../third.snap | 2 +- .../test.rs/swap_indexes/create_a.snap | 8 ++--- .../test.rs/swap_indexes/create_b.snap | 8 ++--- .../test.rs/swap_indexes/create_c.snap | 8 ++--- .../test.rs/swap_indexes/create_d.snap | 8 ++--- .../swap_indexes/first_swap_processed.snap | 22 ++++++------ .../swap_indexes/first_swap_registered.snap | 10 +++--- .../swap_indexes/second_swap_processed.snap | 24 ++++++------- .../third_empty_swap_processed.snap | 24 ++++++------- .../swap_indexes/two_swaps_registered.snap | 12 +++---- .../after_the_index_creation.snap | 8 ++--- .../first_swap_failed.snap | 12 +++---- .../initial_tasks_processed.snap | 8 ++--- .../initial_tasks_enqueued.snap | 3 +- .../task_deletion_done.snap | 2 +- .../task_deletion_enqueued.snap | 3 +- .../task_deletion_processing.snap | 2 +- .../after_restart.snap | 2 +- .../registered_task.snap | 2 +- .../registered_a_task.snap | 3 +- .../after_processing_the_10_tasks.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../five_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_register.snap | 3 +- .../index_creation_failed.snap | 2 +- .../index_creation_failed.snap | 2 +- .../registered_the_first_task.snap | 3 +- .../after_processing_everything.snap | 8 ++--- .../after_removing_the_upgrade_tasks.snap | 8 ++--- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 2 +- .../upgrade_task_failed_again.snap | 4 +-- .../upgrade_task_succeeded.snap | 6 ++-- crates/meilisearch-types/src/error.rs | 1 - crates/meilisearch-types/src/task_view.rs | 4 +-- crates/meilisearch-types/src/tasks.rs | 14 ++++---- crates/meilisearch/src/routes/indexes/mod.rs | 1 - crates/meilisearch/tests/index/errors.rs | 34 +++++-------------- .../meilisearch/tests/index/update_index.rs | 19 +++++++++++ 102 files changed, 300 insertions(+), 320 deletions(-) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index b4b339f09..f56729ce5 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -129,7 +129,7 @@ pub enum KindDump { }, IndexUpdate { primary_key: Option, - new_uid: Option, + uid: Option, }, IndexSwap { swaps: Vec, @@ -211,8 +211,8 @@ impl From for KindDump { KindWithContent::IndexCreation { primary_key, .. } => { KindDump::IndexCreation { primary_key } } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { - KindDump::IndexUpdate { primary_key, new_uid: new_index_uid } + KindWithContent::IndexUpdate { primary_key, new_index_uid: uid, .. } => { + KindDump::IndexUpdate { primary_key, uid } } KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps }, KindWithContent::TaskCancelation { query, tasks } => { diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 790c239d7..9415fa234 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -85,7 +85,7 @@ impl CompatV5ToV6 { v6::Kind::IndexCreation { primary_key } } v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => { - v6::Kind::IndexUpdate { primary_key, new_uid: None } + v6::Kind::IndexUpdate { primary_key, uid: None } } v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion, v5::tasks::TaskContent::DocumentAddition { @@ -141,7 +141,7 @@ impl CompatV5ToV6 { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key, new_uid: None } + v6::Details::IndexInfo { primary_key, uid: None } } v5::Details::DocumentDeletion { received_document_ids, diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 18c665ca3..3f56d63e5 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -197,10 +197,10 @@ impl<'a> Dump<'a> { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, }, - KindDump::IndexUpdate { primary_key, new_uid } => KindWithContent::IndexUpdate { + KindDump::IndexUpdate { primary_key, uid } => KindWithContent::IndexUpdate { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, - new_index_uid: new_uid, + new_index_uid: uid, }, KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps }, KindDump::TaskCancelation { query, tasks } => { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 6d72e4b9f..caef2da39 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key, new_uid } => { - format!("{{ primary_key: {primary_key:?}, new_uid: {new_uid:?} }}") + Details::IndexInfo { primary_key, uid } => { + format!("{{ primary_key: {primary_key:?}, new_uid: {uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap index 48d1ccaab..384db97c8 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap index 4c54de49a..4a9c47047 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap index 74c4c4a33..1247d6029 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap index 411e82ea0..c1777be44 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap index 4c76db95e..a13db4b5e 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap index 7ce0d3ca3..188827723 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap index 603544991..1f136875b 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap index 6dc897dfa..349d9b5eb 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap index 84d6c7878..8f6550ab5 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -54,8 +54,8 @@ doggo: { number_of_documents: 0, field_distribution: {} } ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } -2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } -3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } +2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } +3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap index 1184c197f..a22808394 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap index e3c26b2b3..49bd39572 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap index 4475c71fc..e7879f6b6 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap index 4f9ffb209..b41dc9d4b 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap index eb6b0e7ec..6bbec14b3 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap index 181f0308c..af07454db 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap index 4ac6201a6..062c248ad 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap index 268f463aa..dcb989d9f 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap index 60c041c05..bdcd9be60 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap index e4d9af541..8d1aa089c 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap index 30e8e17a8..9cb42f9e8 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap index 4e3fb5439..62af47a08 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap index 4cabce94b..6b59db7e8 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -34,7 +34,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..035bded64 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "succeeded", @@ -39,7 +39,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "failed", @@ -60,7 +61,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", @@ -81,7 +83,8 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 738df7b5e..7c5469831 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -240,23 +240,19 @@ impl IndexScheduler { // Handle rename if new_index_uid is provided let final_index_uid = if let Some(new_uid) = &new_index_uid { - index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; + if new_uid != &index_uid { + index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; - let mut wtxn = self.env.write_txn()?; + let mut wtxn = self.env.write_txn()?; + self.apply_index_swap( + &mut wtxn, &progress, task.uid, &index_uid, new_uid, true, + )?; + wtxn.commit()?; - // Rename the index - self.index_mapper.rename(&mut wtxn, &index_uid, new_uid)?; - - // Update the task index mappings - let old_tasks = - self.queue.tasks.index_tasks(&wtxn, &index_uid).unwrap_or_default(); - self.queue.tasks.update_index(&mut wtxn, new_uid, |bm| { - *bm |= &old_tasks; - })?; - self.queue.tasks.update_index(&mut wtxn, &index_uid, |bm| bm.clear())?; - wtxn.commit()?; - - new_uid.clone() + new_uid.clone() + } else { + new_uid.clone() + } } else { index_uid.clone() }; @@ -287,7 +283,7 @@ impl IndexScheduler { task.status = Status::Succeeded; task.details = Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }); // if the update processed successfully, we're going to store the new diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap index ed6e75a3d..5f8bd0184 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap index f98e5d308..306d11d6c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap index ae1139c0c..c8565036d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap index 92f24508c..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap index 5f25c2964..4e1f511d7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap index 0006ee8c0..7db4eccbd 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap index 17b69061a..784fa24c0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap index cf2b2b691..de152c46e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap index c8a407554..0b9cc0d2c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap index 0cae69a70..d4bea7912 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "index_a" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap index c5e3e66c8..1e62bab03 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap index 8da1b6ca8..0a8e64b3d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap index 8ee0bfcef..677c0e204 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap index 92f24508c..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap index 21a6a59f7..17c0d6c9a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap index adf9a76fe..ba9ee1c27 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap index a7215f32c..42b9fa416 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap index 1c14b091f..20bfa8041 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap index 5c8082f72..62bf45bfe 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap index a22004697..76d7f8a50 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap index 635491dc1..ab090cac3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap index 1d190baca..f0c0c7652 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap index da91440ab..1ec28ed04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap index 95bc1f7f5..b2607fcfa 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap index 4878bbe28..78eb9cb45 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap index 5a851f373..9975088a5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap index dad7609d2..bdd6e8c14 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap index 39d1b3339..97015c2c6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [5,] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [1,4,5,] -b [0,4,] -c [3,4,5,] -d [2,4,] +a [4,5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -64,7 +64,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap index 5d292fe21..01ea109b0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap index 9327015c4..92a67e022 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -66,8 +66,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap index f85735397..2da41da39 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }} ---------------------------------------------------------------------- ### Status: @@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,] "indexSwap" [4,5,6,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -70,8 +70,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } 6 {uid: 6, details: {"swaps":[]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 6 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap index 46d70dceb..0ce2d02ef 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,5,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap index da9340f3b..cbb84dc48 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -65,7 +65,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","e"]},{"indexes":["d","f"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","e"],"rename":false},{"indexes":["d","f"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap index ee0a12692..378824ba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index 46cbaefc2..bc4b0661e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap index 3b89fe1e7..8961e5280 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 0, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index a861fea12..761a8eedb 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap index d8abc1314..b2492956b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"matchedTasks":2,"deletedTasks":null,"originalFilter":"test_query"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"taskDeletion":1},"indexUids":{}}, stop reason: "stopped after the last task of type `taskDeletion` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap index 1dde1a394..ad1580038 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap index dd1d76f55..ae40c8e00 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap index cf2b2b691..de152c46e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap index f8caaa995..bff14da44 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap index d987d66c0..58e75739d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap index d1369460f..a274275a6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap index 03d4e5b16..788c48d04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap index 136777fcf..96d175a3a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap index 0b4fc96b5..9510f7ee5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap index d938ca288..600602315 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap index 2d936ba68..39da9349d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap index 6f0f9c782..c982042d4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap index 6add8a2a5..7e7f5e0b2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap index 197ed0679..581dd82a3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap index d1369460f..a274275a6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap index 03d4e5b16..788c48d04 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap index 4ece15b13..302ef6200 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index 8feeaf990..4c7ed728c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap index 201680d7a..5bf7186e2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap index 4ece15b13..302ef6200 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index d700dd3db..dcb1d96fc 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -7,10 +7,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index fb682053c..45aed3211 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index abaffbb1b..3fcf91f9f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 9569ecfe3..8bd521e8c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 1d7945023..9e7073b0c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 869d1d0b2..48398a9cb 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 415bb5fdb..ab924c9f7 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -212,7 +212,6 @@ ImmutableApiKeyKey , InvalidRequest , BAD_REQU ImmutableApiKeyUid , InvalidRequest , BAD_REQUEST; ImmutableApiKeyUpdatedAt , InvalidRequest , BAD_REQUEST; ImmutableIndexCreatedAt , InvalidRequest , BAD_REQUEST; -ImmutableIndexUid , InvalidRequest , BAD_REQUEST; ImmutableIndexUpdatedAt , InvalidRequest , BAD_REQUEST; IndexAlreadyExists , InvalidRequest , CONFLICT ; IndexCreationFailed , Internal , INTERNAL_SERVER_ERROR; diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 460ae68d7..dc1106766 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -341,9 +341,9 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key, new_uid } => DetailsView { + Details::IndexInfo { primary_key, uid } => DetailsView { primary_key: Some(primary_key), - new_index_uid: new_uid.clone(), + new_index_uid: uid.clone(), ..DetailsView::default() }, Details::DocumentDeletion { diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index fbdaac9ce..cdd7f5fed 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -279,12 +279,12 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { swaps } => { @@ -358,12 +358,12 @@ impl KindWithContent { } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => { @@ -419,12 +419,12 @@ impl From<&KindWithContent> for Option
{ } KindWithContent::IndexDeletion { .. } => None, KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), new_uid: None }) + Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) } KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - new_uid: new_index_uid.clone(), + uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => None, @@ -678,7 +678,7 @@ pub enum Details { }, IndexInfo { primary_key: Option, - new_uid: Option, + uid: Option, }, DocumentDeletion { provided_ids: usize, diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 632922542..d51ce6c6c 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -288,7 +288,6 @@ fn deny_immutable_fields_index( location: ValuePointerRef, ) -> DeserrJsonError { match field { - "uid" => immutable_field_error(field, accepted, Code::ImmutableIndexUid), "createdAt" => immutable_field_error(field, accepted, Code::ImmutableIndexCreatedAt), "updatedAt" => immutable_field_error(field, accepted, Code::ImmutableIndexUpdatedAt), _ => deserr::take_cf_content(DeserrJsonError::::error::( diff --git a/crates/meilisearch/tests/index/errors.rs b/crates/meilisearch/tests/index/errors.rs index 3bab83955..2f10a5b67 100644 --- a/crates/meilisearch/tests/index/errors.rs +++ b/crates/meilisearch/tests/index/errors.rs @@ -160,36 +160,20 @@ async fn update_index_bad_primary_key() { "###); } -#[actix_rt::test] -async fn update_index_immutable_uid() { - let server = Server::new_shared(); - let index = server.unique_index(); - let (response, code) = index.update_raw(json!({ "uid": "doggo" })).await; - snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" - { - "message": "Immutable field `uid`: expected one of `primaryKey`", - "code": "immutable_index_uid", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#immutable_index_uid" - } - "###); -} - #[actix_rt::test] async fn update_index_immutable_created_at() { let server = Server::new_shared(); let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "createdAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `createdAt`: expected one of `primaryKey`", + "message": "Immutable field `createdAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_created_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_created_at" } - "###); + "#); } #[actix_rt::test] @@ -198,14 +182,14 @@ async fn update_index_immutable_updated_at() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "updatedAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `updatedAt`: expected one of `primaryKey`", + "message": "Immutable field `updatedAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_updated_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_updated_at" } - "###); + "#); } #[actix_rt::test] @@ -214,14 +198,14 @@ async fn update_index_unknown_field() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "doggo": "bork" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Unknown field `doggo`: expected one of `primaryKey`", + "message": "Unknown field `doggo`: expected one of `primaryKey`, `uid`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 8e5837d81..97b31ec68 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -139,6 +139,25 @@ async fn update_index_name() { snapshot!(response.as_object().unwrap().len(), @"4"); } +#[actix_rt::test] +async fn update_index_name_to_itself() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + let (initial_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + let (task, _code) = index.update_raw(json!({ "uid": index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (new_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + // Renaming an index to its own name should not change anything + assert_eq!(initial_response, new_response); +} + #[actix_rt::test] async fn error_update_index_name_to_already_existing_index() { let server = Server::new_shared(); From 81020c7d6d2bb6365e3b15f57cda25b5815e2bd8 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 18:38:21 +0200 Subject: [PATCH 07/38] remove a duplicated test --- .../meilisearch/tests/index/rename_index.rs | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/crates/meilisearch/tests/index/rename_index.rs b/crates/meilisearch/tests/index/rename_index.rs index 793b079f1..23109f272 100644 --- a/crates/meilisearch/tests/index/rename_index.rs +++ b/crates/meilisearch/tests/index/rename_index.rs @@ -390,30 +390,3 @@ async fn rename_index_with_pending_tasks() { let docs = response["results"].as_array().unwrap(); assert!(!docs.is_empty()); // At least the initial document should be there } - -#[actix_rt::test] -async fn rename_index_to_same_name() { - let server = Server::new_shared(); - let index = server.unique_index(); - - // Create index - let (task, code) = index.create(None).await; - assert_eq!(code, 202); - server.wait_task(task.uid()).await.succeeded(); - - // Try to rename to the same name - let body = json!({ "uid": index.uid }); - let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; - - assert_eq!(code, 202); - let response = server.wait_task(task.uid()).await.failed(); - - // Should fail with index already exists error - assert_eq!(response["status"], "failed"); - assert_eq!(response["type"], "indexUpdate"); - assert_eq!(response["error"]["code"], "index_already_exists"); - - // Index should still be accessible with original name - let (_, code) = index.get().await; - assert_eq!(code, 200); -} From 2bab375001700a8fa5b14f9d650687932bb3f5b4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 19:33:15 +0200 Subject: [PATCH 08/38] update the task details again --- crates/dump/src/reader/compat/v5_to_v6.rs | 8 ++-- crates/index-scheduler/src/insta_snapshot.rs | 4 +- .../query_batches_canceled_by/start.snap | 4 +- .../processed_all_tasks.snap | 6 +-- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 6 +-- .../after-advancing-a-bit.snap | 6 +-- .../query_batches_simple/end.snap | 6 +-- .../query_batches_simple/start.snap | 6 +-- .../after-processing-everything.snap | 4 +- .../query_batches_special_rules/start.snap | 4 +- .../query_tasks_canceled_by/start.snap | 4 +- .../processed_all_tasks.snap | 6 +-- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 6 +-- .../tasks_test.rs/query_tasks_simple/end.snap | 6 +-- .../query_tasks_simple/start.snap | 6 +-- .../query_tasks_special_rules/start.snap | 4 +- ...everything_is_successfully_registered.snap | 2 +- .../after_the_second_task_deletion.snap | 3 +- .../task_deletion_have_been_enqueued.snap | 12 ++++-- .../task_deletion_have_been_processed.snap | 6 ++- .../task_queue_is_full.snap | 12 ++++-- .../task_deletion_have_not_been_enqueued.snap | 12 ++++-- .../task_queue_is_full.snap | 12 ++++-- .../src/scheduler/process_batch.rs | 4 +- .../all_tasks_processed.snap | 6 +-- .../before_index_creation.snap | 2 +- .../both_task_succeeded.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 2 +- .../registered_the_third_task.snap | 2 +- .../after_batch_creation.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 4 +- .../processed_the_first_task.snap | 4 +- .../processed_the_second_task.snap | 4 +- .../processed_the_third_task.snap | 4 +- .../registered_the_first_task.snap | 2 +- .../registered_the_second_task.snap | 4 +- .../registered_the_third_task.snap | 4 +- .../first.snap | 2 +- .../fourth.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../registered_the_fourth_task.snap | 2 +- .../registered_the_second_task.snap | 2 +- .../registered_the_third_task.snap | 2 +- .../second.snap | 2 +- .../third.snap | 2 +- .../test.rs/swap_indexes/create_a.snap | 8 ++-- .../test.rs/swap_indexes/create_b.snap | 8 ++-- .../test.rs/swap_indexes/create_c.snap | 8 ++-- .../test.rs/swap_indexes/create_d.snap | 8 ++-- .../swap_indexes/first_swap_processed.snap | 8 ++-- .../swap_indexes/first_swap_registered.snap | 8 ++-- .../swap_indexes/second_swap_processed.snap | 8 ++-- .../third_empty_swap_processed.snap | 8 ++-- .../swap_indexes/two_swaps_registered.snap | 8 ++-- .../after_the_index_creation.snap | 8 ++-- .../first_swap_failed.snap | 8 ++-- .../initial_tasks_processed.snap | 8 ++-- .../initial_tasks_enqueued.snap | 2 +- .../task_deletion_done.snap | 2 +- .../task_deletion_enqueued.snap | 2 +- .../task_deletion_processing.snap | 2 +- .../after_restart.snap | 2 +- .../registered_task.snap | 2 +- .../registered_a_task.snap | 2 +- .../after_processing_the_10_tasks.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../five_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_registering_the_10_tasks.snap | 2 +- .../all_tasks_processed.snap | 2 +- .../processed_the_first_task.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_register.snap | 2 +- .../index_creation_failed.snap | 2 +- .../index_creation_failed.snap | 2 +- .../registered_the_first_task.snap | 2 +- .../after_processing_everything.snap | 8 ++-- .../after_removing_the_upgrade_tasks.snap | 8 ++-- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 2 +- .../upgrade_task_failed_again.snap | 4 +- .../upgrade_task_succeeded.snap | 6 +-- crates/meilisearch-types/src/task_view.rs | 5 ++- crates/meilisearch-types/src/tasks.rs | 42 ++++++++++++------- crates/meilisearch/tests/common/index.rs | 23 ++++++++++ .../meilisearch/tests/index/update_index.rs | 11 +++-- 98 files changed, 269 insertions(+), 213 deletions(-) diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 9415fa234..634604639 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -140,9 +140,11 @@ impl CompatV5ToV6 { v5::Details::Settings { settings } => { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } - v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key, uid: None } - } + v5::Details::IndexInfo { primary_key } => v6::Details::IndexInfo { + primary_key, + new_index_uid: None, + old_index_uid: None, + }, v5::Details::DocumentDeletion { received_document_ids, deleted_documents, diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index caef2da39..e7a068899 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -274,8 +274,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key, uid } => { - format!("{{ primary_key: {primary_key:?}, new_uid: {uid:?} }}") + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => { + format!("{{ primary_key: {primary_key:?}, old_new_uid: {old_index_uid:?}, new_index_uid: {new_index_uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap index 384db97c8..ec05be8f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap index 4a9c47047..a30869296 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap index 1247d6029..9a335acf5 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap index c1777be44..022243e3b 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap index a13db4b5e..08aa4856e 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap index 188827723..fcd523744 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap index 1f136875b..b3443a0ad 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap index 349d9b5eb..2039314cf 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap index 8f6550ab5..1e9eaa775 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap index a22808394..26271c168 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap index 49bd39572..c10efb84a 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap index e7879f6b6..af00b64d0 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap index b41dc9d4b..b56d82bcd 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap index 6bbec14b3..d5e66a7de 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap index af07454db..3e93b86b9 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap index 062c248ad..939f8e9f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap index dcb989d9f..80b1d71be 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), new_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap index bdcd9be60..3949d7c90 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} 2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap index 8d1aa089c..ae4ff5b21 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/queue/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap index 9cb42f9e8..51dee86b1 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap index 62af47a08..96d6c0c9a 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap index 6b59db7e8..99e57fec2 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -35,7 +36,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap index 035bded64..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap @@ -13,7 +13,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -40,7 +41,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -62,7 +64,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -84,7 +87,8 @@ source: crates/index-scheduler/src/queue/test.rs "details": { "IndexInfo": { "primary_key": null, - "uid": null + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 7c5469831..4129c57af 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -283,7 +283,9 @@ impl IndexScheduler { task.status = Status::Succeeded; task.details = Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + new_index_uid: new_index_uid.clone(), + // we only display the old index uid if a rename happened => there is a new_index_uid + old_index_uid: new_index_uid.map(|_| index_uid.clone()), }); // if the update processed successfully, we're going to store the new diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap index 5f8bd0184..e19a94620 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap index 306d11d6c..ae975d843 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap index c8565036d..298ae423f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap index 4e1f511d7..0fe2d8915 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap index 7db4eccbd..50fb4b954 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap index 784fa24c0..7341d77bc 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap index de152c46e..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap index 0b9cc0d2c..aebb02551 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap index d4bea7912..5eb5d37c6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "index_a" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap index 1e62bab03..5aeff9852 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap index 0a8e64b3d..234bdc6a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap index 677c0e204..1b4f9078c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap index 17c0d6c9a..555ac8fed 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap index ba9ee1c27..d259336c0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap index 42b9fa416..0e5f2a994 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap index 20bfa8041..0643dec6b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap index 62bf45bfe..9604b01ad 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap index 76d7f8a50..ae611686b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap index ab090cac3..7b995cf10 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap index f0c0c7652..9ac27f0e7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap index 1ec28ed04..e6f48e5fe 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap index b2607fcfa..8759072a4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap index 78eb9cb45..37afc3447 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap index 9975088a5..683d60cef 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap index bdd6e8c14..d2f999560 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap index 97015c2c6..e7b05d39e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} 5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap index 01ea109b0..ce45dfe50 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap index 92a67e022..0c23e839b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap index 2da41da39..3c9dbc998 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap index 0ce2d02ef..c79a43013 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} 5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap index cbb84dc48..604db5003 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} 4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap index 378824ba4..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index bc4b0661e..b8979eb88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap index 8961e5280..daaeb92be 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 0, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index 761a8eedb..fe9332a88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap index b2492956b..9ab62e8d7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"matchedTasks":2,"deletedTasks":null,"originalFilter":"test_query"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"taskDeletion":1},"indexUids":{}}, stop reason: "stopped after the last task of type `taskDeletion` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap index ad1580038..c2d6d5f07 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap index ae40c8e00..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap index de152c46e..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), new_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap index bff14da44..8cc9bd96b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap index 58e75739d..08cda75e5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap index a274275a6..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap index 788c48d04..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap index 96d175a3a..6bf7eff90 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap index 9510f7ee5..7c667807c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap index 600602315..26a1524a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap index 39da9349d..a9fcb2536 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap index c982042d4..3bc630256 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap index 7e7f5e0b2..86895e12c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap index 581dd82a3..bb769598c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap index a274275a6..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap index 788c48d04..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None, new_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap index 302ef6200..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index 4c7ed728c..72c638b4d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap index 5bf7186e2..6b1e7fa32 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap index 302ef6200..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index dcb1d96fc..ed5a60639 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -7,10 +7,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index 45aed3211..54b7e34e1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), new_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 3fcf91f9f..bfbecb66e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 8bd521e8c..f29fb35c8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 9e7073b0c..57f395473 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 48398a9cb..3bbd4e51e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs ---------------------------------------------------------------------- ### All Tasks: 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), new_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), new_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index dc1106766..df12d56c6 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -341,9 +341,10 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key, uid } => DetailsView { + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => DetailsView { primary_key: Some(primary_key), - new_index_uid: uid.clone(), + new_index_uid: new_index_uid.clone(), + old_index_uid: old_index_uid.clone(), ..DetailsView::default() }, Details::DocumentDeletion { diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index cdd7f5fed..878a74777 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -278,13 +278,16 @@ impl KindWithContent { KindWithContent::SettingsUpdate { new_settings, .. } => { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { swaps } => { @@ -357,13 +360,16 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => { @@ -418,13 +424,16 @@ impl From<&KindWithContent> for Option
{ Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone(), uid: None }) - } - KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_index_uid: None, + old_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { Some(Details::IndexInfo { primary_key: primary_key.clone(), - uid: new_index_uid.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), }) } KindWithContent::IndexSwap { .. } => None, @@ -678,7 +687,8 @@ pub enum Details { }, IndexInfo { primary_key: Option, - uid: Option, + new_index_uid: Option, + old_index_uid: Option, }, DocumentDeletion { provided_ids: usize, diff --git a/crates/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs index 012c9bebe..f8ff5ced9 100644 --- a/crates/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -319,6 +319,24 @@ impl Index<'_, Shared> { } (task, code) } + + pub async fn update_raw_index_fail( + &self, + body: Value, + waiter: &Server, + ) -> (Value, StatusCode) { + let (mut task, code) = self._update_raw(body).await; + if code.is_success() { + task = waiter.wait_task(task.uid()).await; + if task.is_success() { + panic!( + "`update_raw_index_fail` succeeded: {}", + serde_json::to_string_pretty(&task).unwrap() + ); + } + } + (task, code) + } } #[allow(dead_code)] @@ -370,6 +388,11 @@ impl Index<'_, State> { self.service.patch_encoded(url, body, self.encoder).await } + pub(super) async fn _update_raw(&self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); + self.service.patch_encoded(url, body, self.encoder).await + } + pub(super) async fn _delete(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.delete(url).await diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 97b31ec68..5b72d4328 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -162,22 +162,21 @@ async fn update_index_name_to_itself() { async fn error_update_index_name_to_already_existing_index() { let server = Server::new_shared(); let base_index = shared_empty_index().await; - let index = server.unique_index(); - let (task, _code) = index.create(None).await; - server.wait_task(task.uid()).await.succeeded(); + let index = shared_index_with_documents().await; - let (task, _status_code) = index.update_raw(json!({ "uid": base_index.uid })).await; - let task = server.wait_task(task.uid()).await; + let (task, _status_code) = + index.update_raw_index_fail(json!({ "uid": base_index.uid }), server).await; snapshot!(task, @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "[uuid]", + "indexUid": "SHARED_DOCUMENTS", "status": "failed", "type": "indexUpdate", "canceledBy": null, "details": { "primaryKey": null, + "oldIndexUid": "SHARED_DOCUMENTS", "newIndexUid": "EMPTY_INDEX" }, "error": { From b0479eb996308ddacfbee5c12c83df2d6623fd9a Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 12 Aug 2025 15:44:37 +0200 Subject: [PATCH 09/38] make it work with the dump and dumpless upgrade --- crates/meilisearch-types/src/tasks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 878a74777..eb1f61c58 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -173,6 +173,7 @@ pub enum KindWithContent { #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), + #[serde(default)] pub rename: bool, } From 36cac8acf7a94e8eb17c0e0af7ea22c4e48e0a78 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 18 Aug 2025 09:44:40 +0200 Subject: [PATCH 10/38] Update package version --- Cargo.lock | 34 +++++++++++++++++----------------- Cargo.toml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04a3cdb76..5b4e2bccd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "bumpalo", @@ -770,7 +770,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "time", @@ -1774,7 +1774,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "big_s", @@ -2006,7 +2006,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.17.1" +version = "1.18.0" dependencies = [ "tempfile", "thiserror 2.0.12", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.17.1" +version = "1.18.0" dependencies = [ "insta", "levenshtein_automata", @@ -2050,7 +2050,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.17.1" +version = "1.18.0" dependencies = [ "criterion", "serde_json", @@ -2195,7 +2195,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.17.1" +version = "1.18.0" dependencies = [ "arbitrary", "bumpalo", @@ -2995,7 +2995,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "backoff", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.17.1" +version = "1.18.0" dependencies = [ "criterion", "serde_json", @@ -3725,7 +3725,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.17.1" +version = "1.18.0" dependencies = [ "insta", "md5", @@ -3736,7 +3736,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.17.1" +version = "1.18.0" dependencies = [ "actix-cors", "actix-http", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.17.1" +version = "1.18.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3851,7 +3851,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.17.1" +version = "1.18.0" dependencies = [ "actix-web", "anyhow", @@ -3886,7 +3886,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "clap", @@ -3920,7 +3920,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.17.1" +version = "1.18.0" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4483,7 +4483,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.17.1" +version = "1.18.0" dependencies = [ "big_s", "serde_json", @@ -7271,7 +7271,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index bc1c354b7..d68fa222c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.17.1" +version = "1.18.0" authors = [ "Quentin de Quelen ", "Clément Renault ", From c974f0ab0a37a64ebc0375f889fa2fa2bd415a5d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 18 Aug 2025 09:44:55 +0200 Subject: [PATCH 11/38] Update dumpless upgrades --- crates/index-scheduler/src/upgrade/mod.rs | 2 ++ crates/milli/src/update/upgrade/mod.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index a749b31d5..310e48e95 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -40,6 +40,8 @@ pub fn upgrade_index_scheduler( (1, 14, _) => 0, (1, 15, _) => 0, (1, 16, _) => 0, + (1, 17, _) => 0, + (1, 18, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index ecd1cec6c..1152c9234 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -65,6 +65,7 @@ const fn start(from: (u32, u32, u32)) -> Option { (1, 15, _) => function_index!(6), (1, 16, _) => function_index!(7), (1, 17, _) => function_index!(8), + (1, 18, _) => function_index!(8), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, From e13541818a4a41505dcaf23197fdd8c2390aa64b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 18 Aug 2025 09:48:44 +0200 Subject: [PATCH 12/38] Update upgrade tests --- .../upgrade_failure/after_processing_everything.snap | 4 ++-- .../upgrade_failure/register_automatic_upgrade_task.snap | 2 +- .../registered_a_task_while_the_upgrade_task_is_enqueued.snap | 2 +- .../test_failure.rs/upgrade_failure/upgrade_task_failed.snap | 4 ++-- .../upgrade_failure/upgrade_task_failed_again.snap | 4 ++-- .../upgrade_failure/upgrade_task_succeeded.snap | 4 ++-- crates/meilisearch/tests/upgrade/mod.rs | 4 ++-- ...ches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ..._whole_batch_queue_once_everything_has_been_processed.snap | 2 +- ...e_whole_task_queue_once_everything_has_been_processed.snap | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index ed5a60639..6cf99a99f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index ee3cefba4..c99cd3765 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index bfbecb66e..f0fab690e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index f29fb35c8..05c601a41 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 57f395473..9639fd060 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 3bbd4e51e..a14fed454 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index 5d120ba2f..831569ffd 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.1"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.18.0"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.17.2 is higher than the Meilisearch version 1.17.1. Downgrade is not supported"); + snapshot!(err, @"Database version 1.18.1 is higher than the Meilisearch version 1.18.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 8103ceed2..95b23e253 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 81259377c..96a3af5a2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", From a579ea2596add48ddd5f4790c030841da9c3b186 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 18 Aug 2025 10:30:29 +0200 Subject: [PATCH 13/38] Remove useless code --- crates/milli/src/update/upgrade/mod.rs | 6 ++---- crates/milli/src/update/upgrade/v1_16.rs | 19 ------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 1152c9234..718dd41f7 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -8,7 +8,6 @@ use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_15::Latest_V1_14_To_Latest_V1_15; -use v1_16::Latest_V1_16_To_V1_17_0; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; @@ -35,7 +34,6 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, - &Latest_V1_16_To_V1_17_0 {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -64,8 +62,8 @@ const fn start(from: (u32, u32, u32)) -> Option { // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), (1, 16, _) => function_index!(7), - (1, 17, _) => function_index!(8), - (1, 18, _) => function_index!(8), + (1, 17, _) => function_index!(7), + (1, 18, _) => function_index!(7), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs index 02dd136ce..f43efd77d 100644 --- a/crates/milli/src/update/upgrade/v1_16.rs +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -46,22 +46,3 @@ impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { (1, 16, 0) } } - -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_16_To_V1_17_0(); - -impl UpgradeIndex for Latest_V1_16_To_V1_17_0 { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - (1, 17, 0) - } -} From 8b18adee95b860f17e9e3b6f02a331ba4b8d1f7a Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 11:50:31 +0200 Subject: [PATCH 14/38] Add EE license --- LICENSE | 8 +++++++ LICENSE-EE | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 20 ++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 LICENSE-EE diff --git a/LICENSE b/LICENSE index 686f24931..c6ed62f37 100644 --- a/LICENSE +++ b/LICENSE @@ -19,3 +19,11 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +🔒 Meilisearch Enterprise Edition (EE) + +Certain parts of this codebase are not licensed under the MIT license and governed by the Business Source License 1.1. + +See the LICENSE-EE file for details. diff --git a/LICENSE-EE b/LICENSE-EE new file mode 100644 index 000000000..09ae27573 --- /dev/null +++ b/LICENSE-EE @@ -0,0 +1,67 @@ +Business Source License 1.1 – Adapted for Meili SAS +This license is based on the Business Source License version 1.1, as published by MariaDB Corporation Ab. + +Parameters + +Licensor: Meili SAS + +Licensed Work: Any file explicitly marked as “Enterprise Edition (EE)” or “governed by the Business Source License”. + +Additional Use Grant: +You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation. + +Production use of the Licensed Work requires a commercial license agreement with Meilisearch. Contact bonjour@meilisearch.com for licensing. + +Change License: MIT + +Change Date: Four years from the date the Licensed Work is published. + +This License does not apply to any code outside of the Licensed Work, which remains under the MIT license. + +For information about alternative licensing arrangements for the Licensed Work, +please contact bonjour@meilisearch.com or sales@meilisearch.com. + +Notice + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. diff --git a/README.md b/README.md index 40833a0d6..aa371d650 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,26 @@ We also offer a wide range of dedicated guides to all Meilisearch features, such Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced). +## 🧾 Editions & Licensing + +Meilisearch is available in two editions: + +### 🧪 Community Edition (CE) + +- Fully open source under the [MIT license](./LICENSE) +- Core search engine with fast and relevant full-text, semantic or hybrid search +- Free to use for anyone, including commercial usage + +### 🏢 Enterprise Edition (EE) + +- Includes advanced features such as: + - Sharding +- Governed by a [commercial license](./LICENSE-EE) or the [Business Source License 1.1](https://mariadb.com/bsl11) +- Not allowed in production without a commercial agreement with Meilisearch. + - You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation. + +Want access to Enterprise features? → Contact us at [sales@meilisearch.com](maito:sales@meilisearch.com). + ## 📊 Telemetry Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want. From 907055ed08b24d2f134d7b2cfef68d26a5fe19d6 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:28:04 +0200 Subject: [PATCH 15/38] Add `network` to `Task` and `TaskView` --- crates/meilisearch-types/src/task_view.rs | 5 ++++ crates/meilisearch-types/src/tasks.rs | 32 +++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index df12d56c6..cbc29a11b 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -11,6 +11,7 @@ use crate::error::ResponseError; use crate::settings::{Settings, Unchecked}; use crate::tasks::{ serialize_duration, Details, DetailsExportIndexSettings, IndexSwap, Kind, Status, Task, TaskId, + TaskNetwork, }; #[derive(Debug, Clone, PartialEq, Serialize, ToSchema)] @@ -51,6 +52,9 @@ pub struct TaskView { #[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))] #[serde(with = "time::serde::rfc3339::option", default)] pub finished_at: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } impl TaskView { @@ -68,6 +72,7 @@ impl TaskView { enqueued_at: task.enqueued_at, started_at: task.started_at, finished_at: task.finished_at, + network: task.network.clone(), } } } diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index eb1f61c58..2eb389eb9 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -42,6 +42,9 @@ pub struct Task { pub status: Status, pub kind: KindWithContent, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } impl Task { @@ -737,6 +740,35 @@ pub enum Details { }, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(untagged, rename_all = "camelCase")] +pub enum TaskNetwork { + Origin { origin: Origin }, + Remotes { remote_tasks: BTreeMap }, +} +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +pub struct Origin { + pub remote_name: String, + pub task_uid: usize, +} + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RemoteTask { + #[serde(skip_serializing_if = "Option::is_none")] + task_uid: Option, + error: Option, +} + +impl From> for RemoteTask { + fn from(res: Result) -> RemoteTask { + match res { + Ok(task_uid) => RemoteTask { task_uid: Some(task_uid), error: None }, + Err(err) => RemoteTask { task_uid: None, error: Some(err) }, + } + } +} + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] #[schema(rename_all = "camelCase")] pub struct DetailsExportIndexSettings { From e62a807b600a56a45e3a88f2adc6f8313635b76d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:56:47 +0200 Subject: [PATCH 16/38] Add new `milli::update::new::indexer::sharding` module --- crates/milli/src/update/new/indexer/mod.rs | 1 + .../milli/src/update/new/indexer/sharding.rs | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 crates/milli/src/update/new/indexer/sharding.rs diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index a6ba3a919..b24081c02 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -36,6 +36,7 @@ mod guess_primary_key; mod partial_dump; mod post_processing; pub mod settings_changes; +pub mod sharding; mod update_by_function; mod write; diff --git a/crates/milli/src/update/new/indexer/sharding.rs b/crates/milli/src/update/new/indexer/sharding.rs new file mode 100644 index 000000000..c0322f868 --- /dev/null +++ b/crates/milli/src/update/new/indexer/sharding.rs @@ -0,0 +1,22 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::hash::{BuildHasher as _, BuildHasherDefault}; + +pub struct Shards { + pub own: Vec, + pub others: Vec, +} + +impl Shards { + pub fn must_process(&self, docid: &str) -> bool { + let hasher = BuildHasherDefault::::new(); + let to_hash = |shard: &String| hasher.hash_one((shard, docid)); + + let max_hash = self.others.iter().map(to_hash).max().unwrap_or_default(); + + self.own.iter().map(to_hash).any(|hash| hash > max_hash) + } +} From 80ff438402084f4839d549593a2e735004080493 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:36:05 +0200 Subject: [PATCH 17/38] Add `proxy` module to proxy requests to members of a network --- crates/meilisearch/src/routes/indexes/mod.rs | 3 + .../meilisearch/src/routes/indexes/proxy.rs | 410 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 crates/meilisearch/src/routes/indexes/proxy.rs diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index d51ce6c6c..d1946215f 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -30,6 +30,7 @@ use crate::Opt; pub mod documents; pub mod facet_search; +mod proxy; pub mod search; mod search_analytics; #[cfg(test)] @@ -39,6 +40,8 @@ mod settings_analytics; pub mod similar; mod similar_analytics; +pub use proxy::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; + #[derive(OpenApi)] #[openapi( nest( diff --git a/crates/meilisearch/src/routes/indexes/proxy.rs b/crates/meilisearch/src/routes/indexes/proxy.rs new file mode 100644 index 000000000..87858f1f9 --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/proxy.rs @@ -0,0 +1,410 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::collections::BTreeMap; +use std::fs::File; + +use actix_web::http::header::CONTENT_TYPE; +use actix_web::HttpRequest; +use bytes::Bytes; +use index_scheduler::IndexScheduler; +use meilisearch_types::error::ResponseError; +use meilisearch_types::tasks::{Origin, RemoteTask, TaskNetwork}; +use reqwest::StatusCode; +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::error::MeilisearchHttpError; +use crate::routes::indexes::proxy::error::{ProxyDocumentChangeError, ReqwestErrorWithoutUrl}; +use crate::routes::SummarizedTaskView; + +pub enum Body { + File(File), + Inline(T), + None, +} + +impl Body<()> { + pub fn with_file(file: File) -> Self { + Self::File(file) + } + + pub fn none() -> Self { + Self::None + } +} + +/// If necessary, proxies the passed request to the network and update the task description. +/// +/// This function reads the custom headers from the request to determine if must proxy the request or if the request +/// has already been proxied. +/// +/// - when it must proxy the request, the endpoint, method and query params are retrieved from the passed `req`, then the `body` is +/// sent to all remotes of the `network` (except `self`). The response from the remotes are collected to update the passed `task` +/// with the task ids from the task queues of the remotes. +/// - when the request has already been proxied, the custom headers contains information about the remote that created the initial task. +/// This information is copied to the passed task. +pub async fn proxy( + index_scheduler: &IndexScheduler, + index_uid: &str, + req: &HttpRequest, + network: meilisearch_types::network::Network, + body: Body, + task: &meilisearch_types::tasks::Task, +) -> Result<(), MeilisearchHttpError> { + match origin_from_req(req)? { + Some(origin) => { + index_scheduler.set_task_network(task.uid, TaskNetwork::Origin { origin })? + } + None => { + let this = network + .local + .as_deref() + .expect("inconsistent `network.sharding` and `network.self`") + .to_owned(); + + let body = match body { + Body::File(file) => Some(Bytes::from_owner(unsafe { + memmap2::Mmap::map(&file).map_err(|err| { + MeilisearchHttpError::from_milli(err.into(), Some(index_uid.to_owned())) + })? + })), + + Body::Inline(payload) => { + Some(Bytes::copy_from_slice(&serde_json::to_vec(&payload).unwrap())) + } + + Body::None => None, + }; + + let mut in_flight_remote_queries = BTreeMap::new(); + let client = reqwest::ClientBuilder::new() + .connect_timeout(std::time::Duration::from_millis(200)) + .build() + .unwrap(); + + let method = from_old_http_method(req.method()); + + // send payload to all remotes + for (node_name, node) in + network.remotes.into_iter().filter(|(name, _)| name.as_str() != this) + { + let body = body.clone(); + let client = client.clone(); + let api_key = node.write_api_key; + let this = this.clone(); + let method = method.clone(); + let path_and_query = + req.uri().path_and_query().map(|paq| paq.as_str()).unwrap_or("/"); + + in_flight_remote_queries.insert( + node_name, + tokio::spawn({ + let url = format!("{}{}", node.url, path_and_query); + + let url_encoded_this = urlencoding::encode(&this).into_owned(); + let url_encoded_task_uid = task.uid.to_string(); // it's url encoded i promize + + let deadline = + std::time::Instant::now() + std::time::Duration::from_secs(100); + + backoff::future::retry(backoff::ExponentialBackoff::default(), move || { + let url = url.clone(); + let client = client.clone(); + let url_encoded_this = url_encoded_this.clone(); + let url_encoded_task_uid = url_encoded_task_uid.clone(); + let body = body.clone(); + let api_key = api_key.clone(); + let method = method.clone(); + + async move { + try_proxy( + method, + &url, + api_key.as_deref(), + &client, + deadline, + &url_encoded_this, + &url_encoded_task_uid, + body, + ) + .await + } + }) + }), + ); + } + + // wait for all in-flight queries to finish and collect their results + let mut remote_tasks: BTreeMap = BTreeMap::new(); + for (node_name, handle) in in_flight_remote_queries { + match handle.await { + Ok(Ok(res)) => { + let task_uid = res.task_uid; + + remote_tasks.insert(node_name, Ok(task_uid).into()); + } + Ok(Err(error)) => { + remote_tasks.insert(node_name, Err(error.as_response_error()).into()); + } + Err(panic) => match panic.try_into_panic() { + Ok(panic) => { + let msg = match panic.downcast_ref::<&'static str>() { + Some(s) => *s, + None => match panic.downcast_ref::() { + Some(s) => &s[..], + None => "Box", + }, + }; + remote_tasks.insert( + node_name, + Err(ResponseError::from_msg( + msg.to_string(), + meilisearch_types::error::Code::Internal, + )) + .into(), + ); + } + Err(_) => { + tracing::error!("proxy task was unexpectedly cancelled") + } + }, + } + } + + // edit details to contain the return values from the remotes + index_scheduler.set_task_network(task.uid, TaskNetwork::Remotes { remote_tasks })?; + } + } + + Ok(()) +} + +fn from_old_http_method(method: &actix_http::Method) -> reqwest::Method { + match method { + &actix_http::Method::CONNECT => reqwest::Method::CONNECT, + &actix_http::Method::DELETE => reqwest::Method::DELETE, + &actix_http::Method::GET => reqwest::Method::GET, + &actix_http::Method::HEAD => reqwest::Method::HEAD, + &actix_http::Method::OPTIONS => reqwest::Method::OPTIONS, + &actix_http::Method::PATCH => reqwest::Method::PATCH, + &actix_http::Method::POST => reqwest::Method::POST, + &actix_http::Method::PUT => reqwest::Method::PUT, + &actix_http::Method::TRACE => reqwest::Method::TRACE, + method => reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap(), + } +} + +#[allow(clippy::too_many_arguments)] +async fn try_proxy( + method: reqwest::Method, + url: &str, + api_key: Option<&str>, + client: &reqwest::Client, + deadline: std::time::Instant, + url_encoded_this: &str, + url_encoded_task_uid: &str, + body: Option, +) -> Result> { + let timeout = deadline.saturating_duration_since(std::time::Instant::now()); + + let request = client.request(method, url).timeout(timeout); + let request = if let Some(body) = body { request.body(body) } else { request }; + let request = if let Some(api_key) = api_key { request.bearer_auth(api_key) } else { request }; + let request = request.header(PROXY_ORIGIN_TASK_UID_HEADER, url_encoded_task_uid); + let request = request.header(PROXY_ORIGIN_REMOTE_HEADER, url_encoded_this); + let request = request.header(CONTENT_TYPE.as_str(), "application/x-ndjson"); + + let response = request.send().await; + let response = match response { + Ok(response) => response, + Err(error) if error.is_timeout() => { + return Err(backoff::Error::transient(ProxyDocumentChangeError::Timeout)) + } + Err(error) => { + return Err(backoff::Error::transient(ProxyDocumentChangeError::CouldNotSendRequest( + ReqwestErrorWithoutUrl::new(error), + ))) + } + }; + + match response.status() { + status_code if status_code.is_success() => (), + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => { + return Err(backoff::Error::Permanent(ProxyDocumentChangeError::AuthenticationError)) + } + status_code if status_code.is_client_error() => { + let response = parse_error(response).await; + return Err(backoff::Error::transient(ProxyDocumentChangeError::BadRequest { + status_code, + response, + })); + } + status_code if status_code.is_server_error() => { + let response = parse_error(response).await; + return Err(backoff::Error::transient(ProxyDocumentChangeError::RemoteError { + status_code, + response, + })); + } + status_code => { + tracing::warn!( + status_code = status_code.as_u16(), + "remote replied with unexpected status code" + ); + } + } + + let response = match parse_response(response).await { + Ok(response) => response, + Err(response) => { + return Err(backoff::Error::transient( + ProxyDocumentChangeError::CouldNotParseResponse { response }, + )) + } + }; + + Ok(response) +} + +async fn parse_error(response: reqwest::Response) -> Result { + let bytes = match response.bytes().await { + Ok(bytes) => bytes, + Err(error) => return Err(ReqwestErrorWithoutUrl::new(error)), + }; + + Ok(parse_bytes_as_error(&bytes)) +} + +fn parse_bytes_as_error(bytes: &[u8]) -> String { + match serde_json::from_slice::(bytes) { + Ok(value) => value.to_string(), + Err(_) => String::from_utf8_lossy(bytes).into_owned(), + } +} + +async fn parse_response( + response: reqwest::Response, +) -> Result> { + let bytes = match response.bytes().await { + Ok(bytes) => bytes, + Err(error) => return Err(Err(ReqwestErrorWithoutUrl::new(error))), + }; + + match serde_json::from_slice::(&bytes) { + Ok(value) => Ok(value), + Err(_) => Err(Ok(parse_bytes_as_error(&bytes))), + } +} + +mod error { + use meilisearch_types::error::ResponseError; + use reqwest::StatusCode; + + #[derive(Debug, thiserror::Error)] + pub enum ProxyDocumentChangeError { + #[error("{0}")] + CouldNotSendRequest(ReqwestErrorWithoutUrl), + #[error("could not authenticate against the remote host\n - hint: check that the remote instance was registered with a valid API key having the `documents.add` action")] + AuthenticationError, + #[error( + "could not parse response from the remote host as a document addition response{}\n - hint: check that the remote instance is a Meilisearch instance running the same version", + response_from_remote(response) + )] + CouldNotParseResponse { response: Result }, + #[error("remote host responded with code {}{}\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", status_code.as_u16(), response_from_remote(response))] + BadRequest { status_code: StatusCode, response: Result }, + #[error("remote host did not answer before the deadline")] + Timeout, + #[error("remote host responded with code {}{}", status_code.as_u16(), response_from_remote(response))] + RemoteError { status_code: StatusCode, response: Result }, + } + + impl ProxyDocumentChangeError { + pub fn as_response_error(&self) -> ResponseError { + use meilisearch_types::error::Code; + let message = self.to_string(); + let code = match self { + ProxyDocumentChangeError::CouldNotSendRequest(_) => Code::RemoteCouldNotSendRequest, + ProxyDocumentChangeError::AuthenticationError => Code::RemoteInvalidApiKey, + ProxyDocumentChangeError::BadRequest { .. } => Code::RemoteBadRequest, + ProxyDocumentChangeError::Timeout => Code::RemoteTimeout, + ProxyDocumentChangeError::RemoteError { .. } => Code::RemoteRemoteError, + ProxyDocumentChangeError::CouldNotParseResponse { .. } => Code::RemoteBadResponse, + }; + ResponseError::from_msg(message, code) + } + } + + #[derive(Debug, thiserror::Error)] + #[error(transparent)] + pub struct ReqwestErrorWithoutUrl(reqwest::Error); + impl ReqwestErrorWithoutUrl { + pub fn new(inner: reqwest::Error) -> Self { + Self(inner.without_url()) + } + } + + fn response_from_remote(response: &Result) -> String { + match response { + Ok(response) => { + format!(":\n - response from remote: {}", response) + } + Err(error) => { + format!(":\n - additionally, could not retrieve response from remote: {error}") + } + } + } +} + +pub const PROXY_ORIGIN_REMOTE_HEADER: &str = "Meili-Proxy-Origin-Remote"; +pub const PROXY_ORIGIN_TASK_UID_HEADER: &str = "Meili-Proxy-Origin-TaskUid"; + +pub fn origin_from_req(req: &HttpRequest) -> Result, MeilisearchHttpError> { + let (remote_name, task_uid) = match ( + req.headers().get(PROXY_ORIGIN_REMOTE_HEADER), + req.headers().get(PROXY_ORIGIN_TASK_UID_HEADER), + ) { + (None, None) => return Ok(None), + (None, Some(_)) => { + return Err(MeilisearchHttpError::InconsistentOriginHeaders { is_remote_missing: true }) + } + (Some(_), None) => { + return Err(MeilisearchHttpError::InconsistentOriginHeaders { + is_remote_missing: false, + }) + } + (Some(remote_name), Some(task_uid)) => ( + urlencoding::decode(remote_name.to_str().map_err(|err| { + MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_REMOTE_HEADER, + msg: format!("while parsing remote name as UTF-8: {err}"), + } + })?) + .map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_REMOTE_HEADER, + msg: format!("while URL-decoding remote name: {err}"), + })?, + urlencoding::decode(task_uid.to_str().map_err(|err| { + MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while parsing task UID as UTF-8: {err}"), + } + })?) + .map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while URL-decoding task UID: {err}"), + })?, + ), + }; + + let task_uid: usize = + task_uid.parse().map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while parsing the task UID as an integer: {err}"), + })?; + + Ok(Some(Origin { remote_name: remote_name.into_owned(), task_uid })) +} From 9929f798d3fc4c6e093f3eea381d4dbd5f120ac0 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:40:01 +0200 Subject: [PATCH 18/38] network: add `sharding` to `Network` and `writeApiKey` to `Remotes` --- crates/meilisearch/src/routes/network.rs | 63 +++++++++++++++++++----- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 4afa32c09..c966cb897 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -8,7 +8,8 @@ use index_scheduler::IndexScheduler; use itertools::{EitherOrBoth, Itertools}; use meilisearch_types::deserr::DeserrJsonError; use meilisearch_types::error::deserr_codes::{ - InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkUrl, + InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkSharding, + InvalidNetworkUrl, InvalidNetworkWriteApiKey, }; use meilisearch_types::error::ResponseError; use meilisearch_types::features::{Network as DbNetwork, Remote as DbRemote}; @@ -57,9 +58,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { { "self": "ms-0", "remotes": { - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, } })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( @@ -88,9 +89,9 @@ async fn get_network( #[schema(rename_all = "camelCase")] pub struct Remote { #[schema(value_type = Option, example = json!({ - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, }))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -99,6 +100,10 @@ pub struct Remote { #[deserr(default, error = DeserrJsonError)] #[serde(default)] pub search_api_key: Setting, + #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub write_api_key: Setting, } #[derive(Debug, Deserr, ToSchema, Serialize)] @@ -114,6 +119,10 @@ pub struct Network { #[serde(default, rename = "self")] #[deserr(default, rename = "self", error = DeserrJsonError)] pub local: Setting, + #[schema(value_type = Option, example = json!(true))] + #[serde(default)] + #[deserr(default, error = DeserrJsonError)] + pub sharding: Setting, } impl Remote { @@ -136,6 +145,7 @@ impl Remote { Ok(url) })?, search_api_key: self.search_api_key.set(), + write_api_key: self.write_api_key.set(), }) } } @@ -174,9 +184,9 @@ impl Aggregate for PatchNetworkAnalytics { { "self": "ms-0", "remotes": { - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, } })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( @@ -207,6 +217,19 @@ async fn patch_network( Setting::NotSet => old_network.local, }; + let merged_sharding = match new_network.sharding { + Setting::Set(new_sharding) => new_sharding, + Setting::Reset => false, + Setting::NotSet => old_network.sharding, + }; + + if merged_sharding && merged_self.is_none() { + return Err(ResponseError::from_msg( + "`.sharding`: enabling the sharding requires `.self` to be set\n - Hint: Disable `sharding` or set `self` to a value.".into(), + meilisearch_types::error::Code::InvalidNetworkSharding, + )); + } + let merged_remotes = match new_network.remotes { Setting::Set(new_remotes) => { let mut merged_remotes = BTreeMap::new(); @@ -217,9 +240,17 @@ async fn patch_network( { match either_or_both { EitherOrBoth::Both((key, old), (_, Some(new))) => { - let DbRemote { url: old_url, search_api_key: old_search_api_key } = old; + let DbRemote { + url: old_url, + search_api_key: old_search_api_key, + write_api_key: old_write_api_key, + } = old; - let Remote { url: new_url, search_api_key: new_search_api_key } = new; + let Remote { + url: new_url, + search_api_key: new_search_api_key, + write_api_key: new_write_api_key, + } = new; let merged = DbRemote { url: match new_url { @@ -247,6 +278,11 @@ async fn patch_network( Setting::Reset => None, Setting::NotSet => old_search_api_key, }, + write_api_key: match new_write_api_key { + Setting::Set(new_write_api_key) => Some(new_write_api_key), + Setting::Reset => None, + Setting::NotSet => old_write_api_key, + }, }; merged_remotes.insert(key, merged); } @@ -274,7 +310,8 @@ async fn patch_network( &req, ); - let merged_network = DbNetwork { local: merged_self, remotes: merged_remotes }; + let merged_network = + DbNetwork { local: merged_self, remotes: merged_remotes, sharding: merged_sharding }; index_scheduler.put_network(merged_network.clone())?; debug!(returns = ?merged_network, "Patch network"); Ok(HttpResponse::Ok().json(merged_network)) From 069d25dce636e42581aa1408b3b2222ff300ee75 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:41:52 +0200 Subject: [PATCH 19/38] Shard documents --- .../src/scheduler/process_index_operation.rs | 6 +++++ .../update/new/indexer/document_operation.rs | 25 ++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_index_operation.rs b/crates/index-scheduler/src/scheduler/process_index_operation.rs index 62d0e6545..56e2d4693 100644 --- a/crates/index-scheduler/src/scheduler/process_index_operation.rs +++ b/crates/index-scheduler/src/scheduler/process_index_operation.rs @@ -66,6 +66,11 @@ impl IndexScheduler { } IndexOperation::DocumentOperation { index_uid, primary_key, operations, mut tasks } => { progress.update_progress(DocumentOperationProgress::RetrievingConfig); + + let network = self.network(); + + let shards = network.shards(); + // TODO: at some point, for better efficiency we might want to reuse the bumpalo for successive batches. // this is made difficult by the fact we're doing private clones of the index scheduler and sending it // to a fresh thread. @@ -130,6 +135,7 @@ impl IndexScheduler { &mut new_fields_ids_map, &|| must_stop_processing.get(), progress.clone(), + shards.as_ref(), ) .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; diff --git a/crates/milli/src/update/new/indexer/document_operation.rs b/crates/milli/src/update/new/indexer/document_operation.rs index 98faaf145..50364c07c 100644 --- a/crates/milli/src/update/new/indexer/document_operation.rs +++ b/crates/milli/src/update/new/indexer/document_operation.rs @@ -17,6 +17,7 @@ use super::guess_primary_key::retrieve_or_guess_primary_key; use crate::documents::PrimaryKey; use crate::progress::{AtomicPayloadStep, Progress}; use crate::update::new::document::{DocumentContext, Versions}; +use crate::update::new::indexer::sharding::Shards; use crate::update::new::steps::IndexingStep; use crate::update::new::thread_local::MostlySend; use crate::update::new::{DocumentIdentifiers, Insertion, Update}; @@ -71,6 +72,7 @@ impl<'pl> DocumentOperation<'pl> { new_fields_ids_map: &mut FieldsIdsMap, must_stop_processing: &MSP, progress: Progress, + shards: Option<&Shards>, ) -> Result<(DocumentOperationChanges<'pl>, Vec, Option>)> where MSP: Fn() -> bool, @@ -107,6 +109,7 @@ impl<'pl> DocumentOperation<'pl> { &mut bytes, &docids_version_offsets, IndexDocumentsMethod::ReplaceDocuments, + shards, payload, ), Payload::Update(payload) => extract_addition_payload_changes( @@ -120,6 +123,7 @@ impl<'pl> DocumentOperation<'pl> { &mut bytes, &docids_version_offsets, IndexDocumentsMethod::UpdateDocuments, + shards, payload, ), Payload::Deletion(to_delete) => extract_deletion_payload_changes( @@ -127,6 +131,7 @@ impl<'pl> DocumentOperation<'pl> { rtxn, &mut available_docids, &docids_version_offsets, + shards, to_delete, ), }; @@ -173,6 +178,7 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( bytes: &mut u64, main_docids_version_offsets: &hashbrown::HashMap<&'pl str, PayloadOperations<'pl>>, method: IndexDocumentsMethod, + shards: Option<&Shards>, payload: &'pl [u8], ) -> Result>> { use IndexDocumentsMethod::{ReplaceDocuments, UpdateDocuments}; @@ -210,12 +216,20 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( primary_key.as_ref().unwrap() }; + let current_offset = iter.byte_offset(); + let content = &payload[previous_offset..current_offset]; + previous_offset = current_offset; + let external_id = retrieved_primary_key.extract_fields_and_docid(doc, new_fields_ids_map, indexer)?; let external_id = external_id.to_de(); - let current_offset = iter.byte_offset(); - let document_offset = DocumentOffset { content: &payload[previous_offset..current_offset] }; + + if shards.is_some_and(|shards| !shards.must_process(external_id)) { + continue; + } + + let document_offset = DocumentOffset { content }; match main_docids_version_offsets.get(external_id) { None => { @@ -299,8 +313,6 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( }, }, } - - previous_offset = iter.byte_offset(); } if payload.is_empty() { @@ -329,11 +341,16 @@ fn extract_deletion_payload_changes<'s, 'pl: 's>( rtxn: &RoTxn, available_docids: &mut AvailableIds, main_docids_version_offsets: &hashbrown::HashMap<&'s str, PayloadOperations<'pl>>, + shards: Option<&Shards>, to_delete: &'pl [&'pl str], ) -> Result>> { let mut new_docids_version_offsets = hashbrown::HashMap::<&str, PayloadOperations<'pl>>::new(); for external_id in to_delete { + if shards.is_some_and(|shards| !shards.must_process(external_id)) { + continue; + } + match main_docids_version_offsets.get(external_id) { None => { match index.external_documents_ids().get(rtxn, external_id) { From 6e0152921f0660fbd083a9a99d48cf0d244f57c1 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:43:17 +0200 Subject: [PATCH 20/38] Proxy all document tasks to the network when sharding is enabled --- .../src/routes/indexes/documents.rs | 144 ++++++++++++------ 1 file changed, 100 insertions(+), 44 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 5ced4603e..90abddc86 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -45,6 +45,7 @@ use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::indexes::proxy::{proxy, Body}; use crate::routes::indexes::search::fix_sort_query_parameters; use crate::routes::{ get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, @@ -338,6 +339,7 @@ pub async fn delete_document( ) -> Result { let DocumentParam { index_uid, document_id } = path.into_inner(); let index_uid = IndexUid::try_from(index_uid)?; + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -355,10 +357,16 @@ pub async fn delete_document( }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::none(), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!("returns: {:?}", task); Ok(HttpResponse::Accepted().json(task)) } @@ -804,7 +812,6 @@ pub async fn replace_documents( let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; let task = document_addition( - extract_mime_type(&req)?, index_scheduler, index_uid, params.primary_key, @@ -814,8 +821,10 @@ pub async fn replace_documents( uid, dry_run, allow_index_creation, + &req, ) .await?; + debug!(returns = ?task, "Replace documents"); Ok(HttpResponse::Accepted().json(task)) @@ -905,7 +914,6 @@ pub async fn update_documents( let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; let task = document_addition( - extract_mime_type(&req)?, index_scheduler, index_uid, params.primary_key, @@ -915,6 +923,7 @@ pub async fn update_documents( uid, dry_run, allow_index_creation, + &req, ) .await?; debug!(returns = ?task, "Update documents"); @@ -924,7 +933,6 @@ pub async fn update_documents( #[allow(clippy::too_many_arguments)] async fn document_addition( - mime_type: Option, index_scheduler: GuardedData, Data>, index_uid: IndexUid, primary_key: Option, @@ -934,7 +942,11 @@ async fn document_addition( task_id: Option, dry_run: bool, allow_index_creation: bool, + req: &HttpRequest, ) -> Result { + let mime_type = extract_mime_type(req)?; + let network = index_scheduler.network(); + let format = match ( mime_type.as_ref().map(|m| (m.type_().as_str(), m.subtype().as_str())), csv_delimiter, @@ -966,7 +978,7 @@ async fn document_addition( }; let (uuid, mut update_file) = index_scheduler.queue.create_update_file(dry_run)?; - let documents_count = match format { + let res = match format { PayloadType::Ndjson => { let (path, file) = update_file.into_parts(); let file = match file { @@ -981,19 +993,19 @@ async fn document_addition( None => None, }; - let documents_count = tokio::task::spawn_blocking(move || { + let res = tokio::task::spawn_blocking(move || { let documents_count = file.as_ref().map_or(Ok(0), |ntf| { read_ndjson(ntf.as_file()).map_err(MeilisearchHttpError::DocumentFormat) })?; let update_file = file_store::File::from_parts(path, file); - update_file.persist()?; + let update_file = update_file.persist()?; - Ok(documents_count) + Ok((documents_count, update_file)) }) .await?; - Ok(documents_count) + Ok(res) } PayloadType::Json | PayloadType::Csv { delimiter: _ } => { let temp_file = match tempfile() { @@ -1012,16 +1024,16 @@ async fn document_addition( unreachable!("We already wrote the user content into the update file") } }; - // we NEED to persist the file here because we moved the `udpate_file` in another task. - update_file.persist()?; - Ok(documents_count) + // we NEED to persist the file here because we moved the `update_file` in another task. + let file = update_file.persist()?; + Ok((documents_count, file)) }) .await } }; - let documents_count = match documents_count { - Ok(Ok(documents_count)) => documents_count, + let (documents_count, file) = match res { + Ok(Ok((documents_count, file))) => (documents_count, file), // in this case the file has not possibly be persisted. Ok(Err(e)) => return Err(e), Err(e) => { @@ -1063,6 +1075,12 @@ async fn document_addition( } }; + if network.sharding { + if let Some(file) = file { + proxy(&index_scheduler, &index_uid, req, network, Body::with_file(file), &task).await?; + } + } + Ok(task.into()) } @@ -1141,6 +1159,7 @@ pub async fn delete_documents_batch( ) -> Result { debug!(parameters = ?body, "Delete documents by batch"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -1161,10 +1180,16 @@ pub async fn delete_documents_batch( KindWithContent::DocumentDeletion { index_uid: index_uid.to_string(), documents_ids: ids }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(body), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete documents by batch"); Ok(HttpResponse::Accepted().json(task)) @@ -1219,7 +1244,8 @@ pub async fn delete_documents_by_filter( debug!(parameters = ?body, "Delete documents by filter"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let index_uid = index_uid.into_inner(); - let filter = body.into_inner().filter; + let filter = body.into_inner(); + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -1232,17 +1258,30 @@ pub async fn delete_documents_by_filter( ); // we ensure the filter is well formed before enqueuing it - crate::search::parse_filter(&filter, Code::InvalidDocumentFilter, index_scheduler.features())? - .ok_or(MeilisearchHttpError::EmptyFilter)?; + crate::search::parse_filter( + &filter.filter, + Code::InvalidDocumentFilter, + index_scheduler.features(), + )? + .ok_or(MeilisearchHttpError::EmptyFilter)?; - let task = KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr: filter }; + let task = KindWithContent::DocumentDeletionByFilter { + index_uid: index_uid.clone(), + filter_expr: filter.filter.clone(), + }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(filter), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete documents by filter"); Ok(HttpResponse::Accepted().json(task)) @@ -1336,6 +1375,8 @@ pub async fn edit_documents_by_function( .features() .check_edit_documents_by_function("Using the documents edit route")?; + let network = index_scheduler.network(); + let index_uid = IndexUid::try_from(index_uid.into_inner())?; let index_uid = index_uid.into_inner(); let params = params.into_inner(); @@ -1349,13 +1390,12 @@ pub async fn edit_documents_by_function( &req, ); - let DocumentEditionByFunction { filter, context, function } = params; let engine = milli::rhai::Engine::new(); - if let Err(e) = engine.compile(&function) { + if let Err(e) = engine.compile(¶ms.function) { return Err(ResponseError::from_msg(e.to_string(), Code::BadRequest)); } - if let Some(ref filter) = filter { + if let Some(ref filter) = params.filter { // we ensure the filter is well formed before enqueuing it crate::search::parse_filter( filter, @@ -1365,9 +1405,9 @@ pub async fn edit_documents_by_function( .ok_or(MeilisearchHttpError::EmptyFilter)?; } let task = KindWithContent::DocumentEdition { - index_uid, - filter_expr: filter, - context: match context { + index_uid: index_uid.clone(), + filter_expr: params.filter.clone(), + context: match params.context.clone() { Some(Value::Object(m)) => Some(m), None => None, _ => { @@ -1377,15 +1417,21 @@ pub async fn edit_documents_by_function( )) } }, - function, + function: params.function.clone(), }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(params), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Edit documents by function"); Ok(HttpResponse::Accepted().json(task)) @@ -1428,6 +1474,8 @@ pub async fn clear_all_documents( analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; + let network = index_scheduler.network(); + analytics.publish( DocumentsDeletionAggregator { clear_all: true, @@ -1441,10 +1489,18 @@ pub async fn clear_all_documents( let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + + let task = { + let index_scheduler = index_scheduler.clone(); + + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::none(), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete all documents"); Ok(HttpResponse::Accepted().json(task)) From 42ac869c5cb6638c8eb7485e8097d44d5bea9015 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:50:59 +0200 Subject: [PATCH 21/38] Dump support for `network` --- crates/dump/src/lib.rs | 5 ++++- crates/index-scheduler/src/dump.rs | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index f56729ce5..4b12e0322 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -10,7 +10,7 @@ use meilisearch_types::keys::Key; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::settings::Unchecked; use meilisearch_types::tasks::{ - Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId, + Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId, TaskNetwork, }; use meilisearch_types::InstanceUid; use roaring::RoaringBitmap; @@ -94,6 +94,8 @@ pub struct TaskDump { default )] pub finished_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } // A `Kind` specific version made for the dump. If modified you may break the dump. @@ -172,6 +174,7 @@ impl From for TaskDump { enqueued_at: task.enqueued_at, started_at: task.started_at, finished_at: task.finished_at, + network: task.network, } } } diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 3f56d63e5..e0866036b 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -147,6 +147,7 @@ impl<'a> Dump<'a> { canceled_by: task.canceled_by, details: task.details, status: task.status, + network: task.network, kind: match task.kind { KindDump::DocumentImport { primary_key, From 15d34c33e869854bac9c2a6609ad5d89aa021689 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:46:35 +0200 Subject: [PATCH 22/38] file-store: `persist` returns the persisted `File` object --- crates/file-store/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rs index 39ed9482b..d43868c67 100644 --- a/crates/file-store/src/lib.rs +++ b/crates/file-store/src/lib.rs @@ -148,11 +148,10 @@ impl File { Ok(Self { path: PathBuf::new(), file: None }) } - pub fn persist(self) -> Result<()> { - if let Some(file) = self.file { - file.persist(&self.path)?; - } - Ok(()) + pub fn persist(self) -> Result> { + let Some(file) = self.file else { return Ok(None) }; + + Ok(Some(file.persist(&self.path)?)) } } From 56c7f54804f2e4f39c777698d0f72b257b0c70dd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:53:10 +0200 Subject: [PATCH 23/38] `IndexScheduler::set_task_network` --- crates/index-scheduler/src/lib.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 6ad7a8397..797ecc134 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -64,7 +64,7 @@ use meilisearch_types::milli::vector::{ }; use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; -use meilisearch_types::tasks::{KindWithContent, Task}; +use meilisearch_types::tasks::{KindWithContent, Task, TaskNetwork}; use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; @@ -666,6 +666,16 @@ impl IndexScheduler { self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, &processing) } + pub fn set_task_network(&self, task_id: TaskId, network: TaskNetwork) -> Result<()> { + let mut wtxn = self.env.write_txn()?; + let mut task = + self.queue.tasks.get_task(&wtxn, task_id)?.ok_or(Error::TaskNotFound(task_id))?; + task.network = Some(network); + self.queue.tasks.all_tasks.put(&mut wtxn, &task_id, &task)?; + wtxn.commit()?; + Ok(()) + } + /// Return the batches matching the query from the user's point of view along /// with the total number of batches matching the query, ignoring from and limit. /// From bd97a7cc1941de035557ccea038c3f093aa974f0 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:48:33 +0200 Subject: [PATCH 24/38] `IndexScheduler::update_task` now merges the `task.network` and accepts `&mut Task` --- crates/index-scheduler/src/queue/tasks.rs | 23 ++++++++++++++++++++- crates/index-scheduler/src/scheduler/mod.rs | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/index-scheduler/src/queue/tasks.rs b/crates/index-scheduler/src/queue/tasks.rs index 74192232e..83c698ebe 100644 --- a/crates/index-scheduler/src/queue/tasks.rs +++ b/crates/index-scheduler/src/queue/tasks.rs @@ -97,7 +97,22 @@ impl TaskQueue { Ok(self.all_tasks.get(rtxn, &task_id)?) } - pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> { + /// Update the inverted task indexes and write the new value of the task. + /// + /// The passed `task` object typically comes from a previous transaction, so two kinds of modification might have occurred: + /// 1. Modification to the `task` object after loading it from the DB (the purpose of this method is to persist these changes) + /// 2. Modification to the task committed by another transaction in the DB (an annoying consequence of having lost the original + /// transaction from which the `task` instance was deserialized) + /// + /// When calling this function, this `task` is modified to take into account any existing `network` + /// that can have been added since the task was loaded into memory. + /// + /// Any other modification to the task that was committed from the DB since the parameter was pulled from the DB will be overwritten. + /// + /// # Errors + /// + /// - CorruptedTaskQueue: The task doesn't exist in the database + pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &mut Task) -> Result<()> { let old_task = self.get_task(wtxn, task.uid)?.ok_or(Error::CorruptedTaskQueue)?; let reprocessing = old_task.status != Status::Enqueued; @@ -157,6 +172,12 @@ impl TaskQueue { } } + task.network = match (old_task.network, task.network.take()) { + (None, None) => None, + (None, Some(network)) | (Some(network), None) => Some(network), + (Some(_), Some(network)) => Some(network), + }; + self.all_tasks.put(wtxn, &task.uid, task)?; Ok(()) } diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index b2bb90c0b..c57bbf70d 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -268,7 +268,7 @@ impl IndexScheduler { self.queue .tasks - .update_task(&mut wtxn, &task) + .update_task(&mut wtxn, &mut task) .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } if let Some(canceled_by) = canceled_by { @@ -349,7 +349,7 @@ impl IndexScheduler { self.queue .tasks - .update_task(&mut wtxn, &task) + .update_task(&mut wtxn, &mut task) .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } } From ef10c1fb23774da6699445558d679403aa3d60cd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:45:43 +0200 Subject: [PATCH 25/38] Dependency changes --- Cargo.lock | 8 ++++++++ crates/meilisearch/Cargo.toml | 4 +++- crates/milli/Cargo.toml | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5b4e2bccd..33c15f42f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3746,6 +3746,7 @@ dependencies = [ "actix-web-lab", "anyhow", "async-openai", + "backoff", "brotli", "bstr", "build-info", @@ -3989,6 +3990,7 @@ dependencies = [ "time", "tokenizers", "tracing", + "twox-hash", "ureq", "url", "utoipa", @@ -6442,6 +6444,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" + [[package]] name = "typeid" version = "1.0.3" diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 5cbbb6666..c110830c4 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -115,6 +115,9 @@ utoipa-scalar = { version = "0.3.0", optional = true, features = ["actix-web"] } async-openai = { git = "https://github.com/meilisearch/async-openai", branch = "better-error-handling" } secrecy = "0.10.3" actix-web-lab = { version = "0.24.1", default-features = false } +urlencoding = "2.1.3" +backoff = { version = "0.4.0", features = ["tokio"] } + [dev-dependencies] actix-rt = "2.10.0" @@ -125,7 +128,6 @@ manifest-dir-macros = "0.1.18" maplit = "1.0.2" meili-snap = { path = "../meili-snap" } temp-env = "0.3.6" -urlencoding = "2.1.3" wiremock = "0.6.3" yaup = "0.3.1" diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index d94a4d4e1..28da78835 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -109,6 +109,7 @@ utoipa = { version = "5.4.0", features = [ "openapi_extensions", ] } lru = "0.14.0" +twox-hash = { version = "2.1.1", default-features = false, features = ["std", "xxhash3_64", "xxhash64"] } [dev-dependencies] mimalloc = { version = "0.1.47", default-features = false } From 3682b92ee8f4ced5c2159a4281d1e23ad4b8e15d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:53:48 +0200 Subject: [PATCH 26/38] New errors --- crates/meilisearch-types/src/error.rs | 4 ++++ crates/meilisearch/src/error.rs | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index ab924c9f7..f651b2352 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -235,9 +235,11 @@ InvalidDocumentFields , InvalidRequest , BAD_REQU InvalidDocumentRetrieveVectors , InvalidRequest , BAD_REQUEST ; MissingDocumentFilter , InvalidRequest , BAD_REQUEST ; MissingDocumentEditionFunction , InvalidRequest , BAD_REQUEST ; +InconsistentDocumentChangeHeaders , InvalidRequest , BAD_REQUEST ; InvalidDocumentFilter , InvalidRequest , BAD_REQUEST ; InvalidDocumentSort , InvalidRequest , BAD_REQUEST ; InvalidDocumentGeoField , InvalidRequest , BAD_REQUEST ; +InvalidHeaderValue , InvalidRequest , BAD_REQUEST ; InvalidVectorDimensions , InvalidRequest , BAD_REQUEST ; InvalidVectorsType , InvalidRequest , BAD_REQUEST ; InvalidDocumentId , InvalidRequest , BAD_REQUEST ; @@ -266,7 +268,9 @@ InvalidMultiSearchRemote , InvalidRequest , BAD_REQU InvalidMultiSearchWeight , InvalidRequest , BAD_REQUEST ; InvalidNetworkRemotes , InvalidRequest , BAD_REQUEST ; InvalidNetworkSelf , InvalidRequest , BAD_REQUEST ; +InvalidNetworkSharding , InvalidRequest , BAD_REQUEST ; InvalidNetworkSearchApiKey , InvalidRequest , BAD_REQUEST ; +InvalidNetworkWriteApiKey , InvalidRequest , BAD_REQUEST ; InvalidNetworkUrl , InvalidRequest , BAD_REQUEST ; InvalidSearchAttributesToSearchOn , InvalidRequest , BAD_REQUEST ; InvalidSearchAttributesToCrop , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs index 8d4430f07..c90c7c226 100644 --- a/crates/meilisearch/src/error.rs +++ b/crates/meilisearch/src/error.rs @@ -9,6 +9,8 @@ use meilisearch_types::milli::OrderBy; use serde_json::Value; use tokio::task::JoinError; +use crate::routes::indexes::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; + #[derive(Debug, thiserror::Error)] pub enum MeilisearchHttpError { #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}", @@ -80,6 +82,16 @@ pub enum MeilisearchHttpError { MissingSearchHybrid, #[error("Invalid request: both `media` and `vector` parameters are present.")] MediaAndVector, + #[error("Inconsistent `Origin` headers: {} was provided but {} is missing.\n - Hint: Either both headers should be provided, or none of them", if *is_remote_missing { + PROXY_ORIGIN_TASK_UID_HEADER + } else { PROXY_ORIGIN_REMOTE_HEADER }, + if *is_remote_missing { + PROXY_ORIGIN_REMOTE_HEADER + } else { PROXY_ORIGIN_TASK_UID_HEADER } +)] + InconsistentOriginHeaders { is_remote_missing: bool }, + #[error("Invalid value for header {header_name}: {msg}")] + InvalidHeaderValue { header_name: &'static str, msg: String }, } impl MeilisearchHttpError { @@ -124,6 +136,10 @@ impl ErrorCode for MeilisearchHttpError { MeilisearchHttpError::InconsistentFacetOrder { .. } => { Code::InvalidMultiSearchFacetOrder } + MeilisearchHttpError::InconsistentOriginHeaders { .. } => { + Code::InconsistentDocumentChangeHeaders + } + MeilisearchHttpError::InvalidHeaderValue { .. } => Code::InvalidHeaderValue, } } } From d352f33d16ee7d3e9360f20e3177ae4fde823bbd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 14:38:15 +0200 Subject: [PATCH 27/38] Make types Serialize and Deserialize for proxying --- crates/meilisearch/src/routes/indexes/documents.rs | 4 ++-- crates/meilisearch/src/routes/mod.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 90abddc86..54089947e 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1195,7 +1195,7 @@ pub async fn delete_documents_batch( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, ToSchema)] +#[derive(Debug, Deserr, ToSchema, Serialize)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[schema(rename_all = "camelCase")] pub struct DocumentDeletionByFilter { @@ -1287,7 +1287,7 @@ pub async fn delete_documents_by_filter( Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, ToSchema)] +#[derive(Debug, Deserr, ToSchema, Serialize)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct DocumentEditionByFunction { /// A string containing a RHAI function. diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 745ac5824..fd6e777de 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -184,7 +184,7 @@ pub fn is_dry_run(req: &HttpRequest, opt: &Opt) -> Result { .is_some_and(|s| s.to_lowercase() == "true")) } -#[derive(Debug, Serialize, ToSchema)] +#[derive(Debug, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct SummarizedTaskView { /// The task unique identifier. @@ -198,7 +198,10 @@ pub struct SummarizedTaskView { #[serde(rename = "type")] kind: Kind, /// The date on which the task was enqueued. - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde( + serialize_with = "time::serde::rfc3339::serialize", + deserialize_with = "time::serde::rfc3339::deserialize" + )] enqueued_at: OffsetDateTime, } From e33fbcf7b230f1c08f7d95e87ede9ce8dea0da47 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:59:22 +0200 Subject: [PATCH 28/38] Move `meilisearch_types::Network` to its own module --- crates/meilisearch-types/src/features.rs | 19 ---------- crates/meilisearch-types/src/lib.rs | 1 + crates/meilisearch-types/src/network.rs | 47 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 crates/meilisearch-types/src/network.rs diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index ddffb107c..cf66422b2 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use crate::error::{Code, ResponseError}; @@ -32,23 +30,6 @@ pub struct InstanceTogglableFeatures { pub contains_filter: bool, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct Remote { - pub url: String, - #[serde(default)] - pub search_api_key: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] -#[serde(rename_all = "camelCase")] -pub struct Network { - #[serde(default, rename = "self")] - pub local: Option, - #[serde(default)] - pub remotes: BTreeMap, -} - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] #[serde(rename_all = "camelCase")] pub struct ChatCompletionSettings { diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index 9857bfb29..0cb647b0a 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -10,6 +10,7 @@ pub mod index_uid; pub mod index_uid_pattern; pub mod keys; pub mod locales; +pub mod network; pub mod settings; pub mod star_or; pub mod task_view; diff --git a/crates/meilisearch-types/src/network.rs b/crates/meilisearch-types/src/network.rs new file mode 100644 index 000000000..79b3b3808 --- /dev/null +++ b/crates/meilisearch-types/src/network.rs @@ -0,0 +1,47 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::collections::BTreeMap; + +use milli::update::new::indexer::sharding::Shards; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase")] +pub struct Network { + #[serde(default, rename = "self")] + pub local: Option, + #[serde(default)] + pub remotes: BTreeMap, + #[serde(default)] + pub sharding: bool, +} + +impl Network { + pub fn shards(&self) -> Option { + if self.sharding { + let this = self.local.as_deref().expect("Inconsistent `sharding` and `self`"); + let others = self + .remotes + .keys() + .filter(|name| name.as_str() != this) + .map(|name| name.to_owned()) + .collect(); + Some(Shards { own: vec![this.to_owned()], others }) + } else { + None + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Remote { + pub url: String, + #[serde(default)] + pub search_api_key: Option, + #[serde(default)] + pub write_api_key: Option, +} From 9c6c0af0767cc2554273f2b2cb9942cb0779db60 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 13:44:40 +0200 Subject: [PATCH 29/38] Misc churn --- crates/benchmarks/benches/indexing.rs | 30 +++++++++++++++++++ crates/benchmarks/benches/utils.rs | 1 + crates/dump/src/lib.rs | 9 ++++-- crates/dump/src/reader/compat/v5_to_v6.rs | 1 + crates/dump/src/reader/v6/mod.rs | 2 +- crates/dump/src/writer.rs | 3 +- crates/fuzzers/src/bin/fuzz-indexing.rs | 1 + crates/index-scheduler/src/features.rs | 3 +- crates/index-scheduler/src/insta_snapshot.rs | 4 +++ crates/index-scheduler/src/lib.rs | 3 +- crates/index-scheduler/src/queue/mod.rs | 1 + crates/index-scheduler/src/upgrade/mod.rs | 1 + crates/index-scheduler/src/utils.rs | 3 +- crates/meilisearch/src/lib.rs | 1 + crates/meilisearch/src/routes/network.rs | 2 +- .../src/search/federated/perform.rs | 2 +- .../meilisearch/src/search/federated/proxy.rs | 2 +- .../milli/src/search/new/tests/integration.rs | 1 + crates/milli/src/test_index.rs | 3 ++ .../milli/src/update/index_documents/mod.rs | 14 +++++++++ .../milli/tests/search/facet_distribution.rs | 1 + crates/milli/tests/search/mod.rs | 1 + crates/milli/tests/search/query_criteria.rs | 1 + crates/milli/tests/search/typo_tolerance.rs | 1 + 24 files changed, 81 insertions(+), 10 deletions(-) diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 4083b69dd..75805e28c 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -154,6 +154,7 @@ fn indexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -221,6 +222,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -266,6 +268,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -335,6 +338,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -412,6 +416,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -457,6 +462,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -498,6 +504,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -566,6 +573,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -633,6 +641,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -700,6 +709,7 @@ fn indexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -766,6 +776,7 @@ fn reindexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -811,6 +822,7 @@ fn reindexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -879,6 +891,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -956,6 +969,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1002,6 +1016,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1044,6 +1059,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1111,6 +1127,7 @@ fn indexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1177,6 +1194,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1222,6 +1240,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1290,6 +1309,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1404,6 +1424,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1449,6 +1470,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1490,6 +1512,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1580,6 +1603,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1671,6 +1695,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1754,6 +1779,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1821,6 +1847,7 @@ fn indexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1887,6 +1914,7 @@ fn reindexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1932,6 +1960,7 @@ fn reindexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2000,6 +2029,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 0abbd6c71..44066767a 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -123,6 +123,7 @@ pub fn base_setup(conf: &Conf) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index 4b12e0322..b374131f5 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -254,11 +254,12 @@ pub(crate) mod test { use maplit::{btreemap, btreeset}; use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats}; use meilisearch_types::facet_values_sort::FacetValuesSort; - use meilisearch_types::features::{Network, Remote, RuntimeTogglableFeatures}; + use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::keys::{Action, Key}; use meilisearch_types::milli::update::Setting; use meilisearch_types::milli::{self, FilterableAttributesRule}; + use meilisearch_types::network::{Network, Remote}; use meilisearch_types::settings::{Checked, FacetingSettings, Settings}; use meilisearch_types::task_view::DetailsView; use meilisearch_types::tasks::{BatchStopReason, Details, Kind, Status}; @@ -387,6 +388,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-11 0:00 UTC), started_at: Some(datetime!(2022-11-20 0:00 UTC)), finished_at: Some(datetime!(2022-11-21 0:00 UTC)), + network: None, }, None, ), @@ -411,6 +413,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-11 0:00 UTC), started_at: None, finished_at: None, + network: None, }, Some(vec![ json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(), @@ -430,6 +433,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-15 0:00 UTC), started_at: None, finished_at: None, + network: None, }, None, ), @@ -542,7 +546,8 @@ pub(crate) mod test { fn create_test_network() -> Network { Network { local: Some("myself".to_string()), - remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()) }}, + remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }}, + sharding: false, } } diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 634604639..cdfa6847e 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -163,6 +163,7 @@ impl CompatV5ToV6 { enqueued_at: task_view.enqueued_at, started_at: task_view.started_at, finished_at: task_view.finished_at, + network: None, }; (task, content_file) diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 9bc4b33c5..d361ab0b1 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch; pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; -pub type Network = meilisearch_types::features::Network; +pub type Network = meilisearch_types::network::Network; pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 1d41b6aa5..ad6d1651b 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -5,8 +5,9 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; -use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; +use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; +use meilisearch_types::network::Network; use meilisearch_types::settings::{Checked, Settings}; use meilisearch_types::webhooks::WebhooksDumpView; use serde_json::{Map, Value}; diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index ec1f96fd5..7ca4d3092 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -129,6 +129,7 @@ fn main() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index dee665458..c9a4af9af 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -1,8 +1,9 @@ use std::sync::{Arc, RwLock}; -use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures}; +use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{SerdeJson, Str}; use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls}; +use meilisearch_types::network::Network; use crate::error::FeatureNotEnabledError; use crate::Result; diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index e7a068899..df043ad87 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -230,6 +230,7 @@ pub fn snapshot_task(task: &Task) -> String { details, status, kind, + network, } = task; snap.push('{'); snap.push_str(&format!("uid: {uid}, ")); @@ -247,6 +248,9 @@ pub fn snapshot_task(task: &Task) -> String { snap.push_str(&format!("details: {}, ", &snapshot_details(details))); } snap.push_str(&format!("kind: {kind:?}")); + if let Some(network) = network { + snap.push_str(&format!("network: {network:?}, ")) + } snap.push('}'); snap diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 797ecc134..ef02d9c3b 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -52,7 +52,7 @@ use flate2::bufread::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; use meilisearch_types::features::{ - ChatCompletionSettings, InstanceTogglableFeatures, Network, RuntimeTogglableFeatures, + ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures, }; use meilisearch_types::heed::byteorder::BE; use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128}; @@ -63,6 +63,7 @@ use meilisearch_types::milli::vector::{ Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment, }; use meilisearch_types::milli::{self, Index}; +use meilisearch_types::network::Network; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task, TaskNetwork}; use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; diff --git a/crates/index-scheduler/src/queue/mod.rs b/crates/index-scheduler/src/queue/mod.rs index 92de10fe1..4f02edfb2 100644 --- a/crates/index-scheduler/src/queue/mod.rs +++ b/crates/index-scheduler/src/queue/mod.rs @@ -279,6 +279,7 @@ impl Queue { details: kind.default_details(), status: Status::Enqueued, kind: kind.clone(), + network: None, }; // For deletion and cancelation tasks, we want to make extra sure that they // don't attempt to delete/cancel tasks that are newer than themselves. diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index 310e48e95..1743c3940 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -91,6 +91,7 @@ pub fn upgrade_index_scheduler( details: Some(Details::UpgradeDatabase { from, to }), status: Status::Enqueued, kind: KindWithContent::UpgradeDatabase { from }, + network: None, }, )?; wtxn.commit()?; diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 159e8f3d3..2617aba99 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -1,6 +1,5 @@ //! Utility functions on the DBs. Mainly getter and setters. -use crate::milli::progress::EmbedderStats; use std::collections::{BTreeSet, HashSet}; use std::ops::Bound; use std::sync::Arc; @@ -15,6 +14,7 @@ use meilisearch_types::tasks::{ use roaring::RoaringBitmap; use time::OffsetDateTime; +use crate::milli::progress::EmbedderStats; use crate::{Error, Result, Task, TaskId, BEI128}; /// This structure contains all the information required to write a batch in the database without reading the tasks. @@ -377,6 +377,7 @@ impl crate::IndexScheduler { details, status, kind, + network: _, } = task; assert_eq!(uid, task.uid); if task.status != Status::Enqueued { diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index ca9bb6f50..c24f81742 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -628,6 +628,7 @@ fn import_dump( &mut new_fields_ids_map, &|| false, // never stop processing a dump progress.clone(), + None, )?; let operation_stats = operation_stats.pop().unwrap(); diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index c966cb897..f4119b0ad 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -12,9 +12,9 @@ use meilisearch_types::error::deserr_codes::{ InvalidNetworkUrl, InvalidNetworkWriteApiKey, }; use meilisearch_types::error::ResponseError; -use meilisearch_types::features::{Network as DbNetwork, Remote as DbRemote}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; +use meilisearch_types::network::{Network as DbNetwork, Remote as DbRemote}; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 3c80c22e3..6fb8bcd43 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -10,11 +10,11 @@ use actix_http::StatusCode; use index_scheduler::{IndexScheduler, RoFeatures}; use itertools::Itertools; use meilisearch_types::error::ResponseError; -use meilisearch_types::features::{Network, Remote}; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; use meilisearch_types::milli::vector::Embedding; use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget, DEFAULT_VALUES_PER_FACET}; +use meilisearch_types::network::{Network, Remote}; use roaring::RoaringBitmap; use tokio::task::JoinHandle; diff --git a/crates/meilisearch/src/search/federated/proxy.rs b/crates/meilisearch/src/search/federated/proxy.rs index bf954693c..6da92ca9e 100644 --- a/crates/meilisearch/src/search/federated/proxy.rs +++ b/crates/meilisearch/src/search/federated/proxy.rs @@ -1,6 +1,6 @@ pub use error::ProxySearchError; use error::ReqwestErrorWithoutUrl; -use meilisearch_types::features::Remote; +use meilisearch_types::network::Remote; use rand::Rng as _; use reqwest::{Client, Response, StatusCode}; use serde::de::DeserializeOwned; diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index 6b8c25ab8..60e471579 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -76,6 +76,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6e34961e7..b831d55cc 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -84,6 +84,7 @@ impl TempIndex { &mut new_fields_ids_map, &|| false, Progress::default(), + None, )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -167,6 +168,7 @@ impl TempIndex { &mut new_fields_ids_map, &|| false, Progress::default(), + None, )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -242,6 +244,7 @@ fn aborting_indexation() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 658ff1923..099879382 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -1977,6 +1977,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2029,6 +2030,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2117,6 +2119,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2306,6 +2309,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2369,6 +2373,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2423,6 +2428,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2476,6 +2482,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2531,6 +2538,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2591,6 +2599,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2644,6 +2653,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2697,6 +2707,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2908,6 +2919,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2968,6 +2980,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -3025,6 +3038,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index cc1b85369..2f347516b 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -59,6 +59,7 @@ fn test_facet_distribution_with_no_facet_values() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index 578a22009..f88f6c392 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -97,6 +97,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 3f8134085..f92cf9e51 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -329,6 +329,7 @@ fn criteria_ascdesc() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index 95ff85165..d44848353 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -138,6 +138,7 @@ fn test_typo_disabled_on_word() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); From 484dbf8c064570a2abb5d0d59f39936cbbfdbf16 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 15:31:29 +0200 Subject: [PATCH 30/38] Update snap --- crates/meilisearch/tests/network/mod.rs | 199 +++++++++++------- .../meilisearch/tests/search/multi/proxy.rs | 81 ++++--- 2 files changed, 172 insertions(+), 108 deletions(-) diff --git a/crates/meilisearch/tests/network/mod.rs b/crates/meilisearch/tests/network/mod.rs index 60f73ed40..e25656eb0 100644 --- a/crates/meilisearch/tests/network/mod.rs +++ b/crates/meilisearch/tests/network/mod.rs @@ -46,7 +46,7 @@ async fn errors_on_param() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "Unknown field `selfie`: expected one of `remotes`, `self`", + "message": "Unknown field `selfie`: expected one of `remotes`, `self`, `sharding`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" @@ -149,7 +149,7 @@ async fn errors_on_param() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "Unknown field `doggo` inside `.remotes.new`: expected one of `url`, `searchApiKey`", + "message": "Unknown field `doggo` inside `.remotes.new`: expected one of `url`, `searchApiKey`, `writeApiKey`", "code": "invalid_network_remotes", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_network_remotes" @@ -192,9 +192,11 @@ async fn errors_on_param() { "remotes": { "kefir": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null } - } + }, + "sharding": false } "###); let (response, code) = server @@ -266,7 +268,8 @@ async fn auth() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "master", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -274,11 +277,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "master", - "remotes": {} -} -"###); + { + "self": "master", + "remotes": {}, + "sharding": false + } + "###); // try get with get permission server.use_api_key(get_network_key.as_str().unwrap()); @@ -286,11 +290,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "master", - "remotes": {} -} -"###); + { + "self": "master", + "remotes": {}, + "sharding": false + } + "###); // try update with update permission server.use_api_key(update_network_key.as_str().unwrap()); @@ -303,11 +308,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "api_key", - "remotes": {} -} -"###); + { + "self": "api_key", + "remotes": {}, + "sharding": false + } + "###); // try with the other's permission let (response, code) = server.get_network().await; @@ -383,7 +389,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": null, - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -393,7 +400,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "myself", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -417,13 +425,16 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "foo" + "searchApiKey": "foo", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -443,13 +454,16 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -470,17 +484,21 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -498,13 +516,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -518,13 +539,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -538,13 +562,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -553,60 +580,69 @@ async fn get_and_set_network() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // still doing nothing let (response, code) = server.set_network(json!({"remotes": {}})).await; meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // good time to check GET let (response, code) = server.get_network().await; meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // deleting everything let (response, code) = server @@ -619,7 +655,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "thy", - "remotes": {} + "remotes": {}, + "sharding": false } "###); } diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 2b1623ff8..ac997f859 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -132,7 +132,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -140,7 +141,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; @@ -148,7 +150,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms2", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -934,7 +937,8 @@ async fn error_unregistered_remote() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -942,7 +946,8 @@ async fn error_unregistered_remote() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1052,7 +1057,8 @@ async fn error_no_weighted_score() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1060,7 +1066,8 @@ async fn error_no_weighted_score() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1185,7 +1192,8 @@ async fn error_bad_response() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1193,7 +1201,8 @@ async fn error_bad_response() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1322,7 +1331,8 @@ async fn error_bad_request() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1330,7 +1340,8 @@ async fn error_bad_request() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1452,7 +1463,8 @@ async fn error_bad_request_facets_by_index() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1460,7 +1472,8 @@ async fn error_bad_request_facets_by_index() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1593,7 +1606,8 @@ async fn error_bad_request_facets_by_index_facet() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1601,7 +1615,8 @@ async fn error_bad_request_facets_by_index_facet() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1743,7 +1758,8 @@ async fn error_remote_does_not_answer() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1751,7 +1767,8 @@ async fn error_remote_does_not_answer() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1944,7 +1961,8 @@ async fn error_remote_404() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1952,7 +1970,8 @@ async fn error_remote_404() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2139,7 +2158,8 @@ async fn error_remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2147,7 +2167,8 @@ async fn error_remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2299,7 +2320,8 @@ async fn remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2307,7 +2329,8 @@ async fn remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2454,7 +2477,8 @@ async fn error_remote_500() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2462,7 +2486,8 @@ async fn error_remote_500() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2633,7 +2658,8 @@ async fn error_remote_500_once() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2641,7 +2667,8 @@ async fn error_remote_500_once() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); From 385acbbcd27b12d4c42e2ae37e4c5c1ab1f02864 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 29 Jul 2025 17:52:24 +0200 Subject: [PATCH 31/38] Don't always hardcode Content-Type in proxy --- .../meilisearch/src/routes/indexes/proxy.rs | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/proxy.rs b/crates/meilisearch/src/routes/indexes/proxy.rs index 87858f1f9..01bb8b021 100644 --- a/crates/meilisearch/src/routes/indexes/proxy.rs +++ b/crates/meilisearch/src/routes/indexes/proxy.rs @@ -21,14 +21,14 @@ use crate::routes::indexes::proxy::error::{ProxyDocumentChangeError, ReqwestErro use crate::routes::SummarizedTaskView; pub enum Body { - File(File), + NdJsonPayload(File), Inline(T), None, } impl Body<()> { pub fn with_file(file: File) -> Self { - Self::File(file) + Self::NdJsonPayload(file) } pub fn none() -> Self { @@ -65,8 +65,15 @@ pub async fn proxy( .expect("inconsistent `network.sharding` and `network.self`") .to_owned(); + let content_type = match &body { + // for file bodies, force x-ndjson + Body::NdJsonPayload(_) => Some(b"application/x-ndjson".as_slice()), + // otherwise get content type from request + _ => req.headers().get(CONTENT_TYPE).map(|h| h.as_bytes()), + }; + let body = match body { - Body::File(file) => Some(Bytes::from_owner(unsafe { + Body::NdJsonPayload(file) => Some(Bytes::from_owner(unsafe { memmap2::Mmap::map(&file).map_err(|err| { MeilisearchHttpError::from_milli(err.into(), Some(index_uid.to_owned())) })? @@ -110,11 +117,15 @@ pub async fn proxy( let deadline = std::time::Instant::now() + std::time::Duration::from_secs(100); + let content_type = content_type.map(|b| b.to_owned()); + backoff::future::retry(backoff::ExponentialBackoff::default(), move || { let url = url.clone(); let client = client.clone(); let url_encoded_this = url_encoded_this.clone(); let url_encoded_task_uid = url_encoded_task_uid.clone(); + let content_type = content_type.clone(); + let body = body.clone(); let api_key = api_key.clone(); let method = method.clone(); @@ -123,6 +134,7 @@ pub async fn proxy( try_proxy( method, &url, + content_type.as_deref(), api_key.as_deref(), &client, deadline, @@ -201,6 +213,7 @@ fn from_old_http_method(method: &actix_http::Method) -> reqwest::Method { async fn try_proxy( method: reqwest::Method, url: &str, + content_type: Option<&[u8]>, api_key: Option<&str>, client: &reqwest::Client, deadline: std::time::Instant, @@ -215,7 +228,11 @@ async fn try_proxy( let request = if let Some(api_key) = api_key { request.bearer_auth(api_key) } else { request }; let request = request.header(PROXY_ORIGIN_TASK_UID_HEADER, url_encoded_task_uid); let request = request.header(PROXY_ORIGIN_REMOTE_HEADER, url_encoded_this); - let request = request.header(CONTENT_TYPE.as_str(), "application/x-ndjson"); + let request = if let Some(content_type) = content_type { + request.header(CONTENT_TYPE.as_str(), content_type) + } else { + request + }; let response = request.send().await; let response = match response { From d550b90c60cca365dba8101afb062ebdb9f3a559 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Jul 2025 17:32:54 +0200 Subject: [PATCH 32/38] Adjust timeouts --- .../meilisearch/src/routes/indexes/proxy.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/proxy.rs b/crates/meilisearch/src/routes/indexes/proxy.rs index 01bb8b021..e22af8dee 100644 --- a/crates/meilisearch/src/routes/indexes/proxy.rs +++ b/crates/meilisearch/src/routes/indexes/proxy.rs @@ -88,7 +88,7 @@ pub async fn proxy( let mut in_flight_remote_queries = BTreeMap::new(); let client = reqwest::ClientBuilder::new() - .connect_timeout(std::time::Duration::from_millis(200)) + .connect_timeout(std::time::Duration::from_secs(3)) .build() .unwrap(); @@ -114,12 +114,13 @@ pub async fn proxy( let url_encoded_this = urlencoding::encode(&this).into_owned(); let url_encoded_task_uid = task.uid.to_string(); // it's url encoded i promize - let deadline = - std::time::Instant::now() + std::time::Duration::from_secs(100); - let content_type = content_type.map(|b| b.to_owned()); - backoff::future::retry(backoff::ExponentialBackoff::default(), move || { + let backoff = backoff::ExponentialBackoffBuilder::new() + .with_max_elapsed_time(Some(std::time::Duration::from_secs(25))) + .build(); + + backoff::future::retry(backoff, move || { let url = url.clone(); let client = client.clone(); let url_encoded_this = url_encoded_this.clone(); @@ -137,7 +138,6 @@ pub async fn proxy( content_type.as_deref(), api_key.as_deref(), &client, - deadline, &url_encoded_this, &url_encoded_task_uid, body, @@ -216,14 +216,11 @@ async fn try_proxy( content_type: Option<&[u8]>, api_key: Option<&str>, client: &reqwest::Client, - deadline: std::time::Instant, url_encoded_this: &str, url_encoded_task_uid: &str, body: Option, ) -> Result> { - let timeout = deadline.saturating_duration_since(std::time::Instant::now()); - - let request = client.request(method, url).timeout(timeout); + let request = client.request(method, url).timeout(std::time::Duration::from_secs(30)); let request = if let Some(body) = body { request.body(body) } else { request }; let request = if let Some(api_key) = api_key { request.bearer_auth(api_key) } else { request }; let request = request.header(PROXY_ORIGIN_TASK_UID_HEADER, url_encoded_task_uid); @@ -254,7 +251,7 @@ async fn try_proxy( } status_code if status_code.is_client_error() => { let response = parse_error(response).await; - return Err(backoff::Error::transient(ProxyDocumentChangeError::BadRequest { + return Err(backoff::Error::Permanent(ProxyDocumentChangeError::BadRequest { status_code, response, })); From 85feb3a26c82bf6b3c67392f913f04f9f7bc259d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 30 Jul 2025 17:33:01 +0200 Subject: [PATCH 33/38] Rename Body::with_file --- crates/meilisearch/src/routes/indexes/documents.rs | 10 +++++++++- crates/meilisearch/src/routes/indexes/proxy.rs | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 54089947e..d188a9867 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1077,7 +1077,15 @@ async fn document_addition( if network.sharding { if let Some(file) = file { - proxy(&index_scheduler, &index_uid, req, network, Body::with_file(file), &task).await?; + proxy( + &index_scheduler, + &index_uid, + req, + network, + Body::with_ndjson_payload(file), + &task, + ) + .await?; } } diff --git a/crates/meilisearch/src/routes/indexes/proxy.rs b/crates/meilisearch/src/routes/indexes/proxy.rs index e22af8dee..a1106b974 100644 --- a/crates/meilisearch/src/routes/indexes/proxy.rs +++ b/crates/meilisearch/src/routes/indexes/proxy.rs @@ -27,7 +27,7 @@ pub enum Body { } impl Body<()> { - pub fn with_file(file: File) -> Self { + pub fn with_ndjson_payload(file: File) -> Self { Self::NdJsonPayload(file) } From 65c212d1fd439abcf12ad37f0fadd6b8dc5d31f1 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 31 Jul 2025 17:32:45 +0200 Subject: [PATCH 34/38] camel case the fields in "origin" --- crates/meilisearch-types/src/tasks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 2eb389eb9..d2e11b529 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -747,6 +747,7 @@ pub enum TaskNetwork { Remotes { remote_tasks: BTreeMap }, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(untagged, rename_all = "camelCase")] pub struct Origin { pub remote_name: String, pub task_uid: usize, From 601785692fb0cd82c6dac71d6e0688dfe167b5dd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 18 Aug 2025 09:24:34 +0200 Subject: [PATCH 35/38] Remove erroneous `untagged` annotation --- crates/meilisearch-types/src/tasks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index d2e11b529..d0f668255 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -747,7 +747,7 @@ pub enum TaskNetwork { Remotes { remote_tasks: BTreeMap }, } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] -#[serde(untagged, rename_all = "camelCase")] +#[serde(rename_all = "camelCase")] pub struct Origin { pub remote_name: String, pub task_uid: usize, From cbd2bdf0fa72c8c9c77ba594a60860122b2cb10d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 18 Aug 2025 10:43:58 +0200 Subject: [PATCH 36/38] Fix snapshots --- crates/meilisearch/tests/search/multi/proxy.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index ac997f859..c435a491e 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -447,7 +447,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -455,7 +456,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; @@ -463,7 +465,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms2", - "remotes": {} + "remotes": {}, + "sharding": false } "###); From 6fb3cf95e4724d1ee5e243c9395f908e7d02194b Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 19 Aug 2025 14:47:27 +0200 Subject: [PATCH 37/38] Move EE files into EE modules --- crates/dump/src/lib.rs | 2 +- crates/dump/src/reader/v6/mod.rs | 2 +- crates/dump/src/writer.rs | 2 +- crates/index-scheduler/src/features.rs | 2 +- crates/index-scheduler/src/lib.rs | 2 +- crates/meilisearch-types/src/enterprise_edition/mod.rs | 6 ++++++ .../src/{ => enterprise_edition}/network.rs | 2 +- crates/meilisearch-types/src/lib.rs | 2 +- crates/meilisearch/src/routes/indexes/documents.rs | 2 +- .../src/routes/indexes/enterprise_edition/mod.rs | 6 ++++++ .../src/routes/indexes/{ => enterprise_edition}/proxy.rs | 6 ++++-- crates/meilisearch/src/routes/indexes/mod.rs | 4 ++-- crates/meilisearch/src/routes/network.rs | 2 +- crates/meilisearch/src/search/federated/perform.rs | 2 +- crates/meilisearch/src/search/federated/proxy.rs | 2 +- crates/milli/src/update/new/indexer/document_operation.rs | 2 +- .../milli/src/update/new/indexer/enterprise_edition/mod.rs | 6 ++++++ .../update/new/indexer/{ => enterprise_edition}/sharding.rs | 0 crates/milli/src/update/new/indexer/mod.rs | 2 +- 19 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 crates/meilisearch-types/src/enterprise_edition/mod.rs rename crates/meilisearch-types/src/{ => enterprise_edition}/network.rs (95%) create mode 100644 crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs rename crates/meilisearch/src/routes/indexes/{ => enterprise_edition}/proxy.rs (98%) create mode 100644 crates/milli/src/update/new/indexer/enterprise_edition/mod.rs rename crates/milli/src/update/new/indexer/{ => enterprise_edition}/sharding.rs (100%) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index b374131f5..a2b72e0e5 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -253,13 +253,13 @@ pub(crate) mod test { use big_s::S; use maplit::{btreemap, btreeset}; use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats}; + use meilisearch_types::enterprise_edition::network::{Network, Remote}; use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::keys::{Action, Key}; use meilisearch_types::milli::update::Setting; use meilisearch_types::milli::{self, FilterableAttributesRule}; - use meilisearch_types::network::{Network, Remote}; use meilisearch_types::settings::{Checked, FacetingSettings, Settings}; use meilisearch_types::task_view::DetailsView; use meilisearch_types::tasks::{BatchStopReason, Details, Kind, Status}; diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index d361ab0b1..75ff2ebe6 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch; pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; -pub type Network = meilisearch_types::network::Network; +pub type Network = meilisearch_types::enterprise_edition::network::Network; pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index ad6d1651b..1f8000a50 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -5,9 +5,9 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; +use meilisearch_types::enterprise_edition::network::Network; use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; -use meilisearch_types::network::Network; use meilisearch_types::settings::{Checked, Settings}; use meilisearch_types::webhooks::WebhooksDumpView; use serde_json::{Map, Value}; diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index c9a4af9af..1b01f89de 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -1,9 +1,9 @@ use std::sync::{Arc, RwLock}; +use meilisearch_types::enterprise_edition::network::Network; use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{SerdeJson, Str}; use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls}; -use meilisearch_types::network::Network; use crate::error::FeatureNotEnabledError; use crate::Result; diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index ef02d9c3b..d6c150dbb 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -51,6 +51,7 @@ pub use features::RoFeatures; use flate2::bufread::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; +use meilisearch_types::enterprise_edition::network::Network; use meilisearch_types::features::{ ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures, }; @@ -63,7 +64,6 @@ use meilisearch_types::milli::vector::{ Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment, }; use meilisearch_types::milli::{self, Index}; -use meilisearch_types::network::Network; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task, TaskNetwork}; use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; diff --git a/crates/meilisearch-types/src/enterprise_edition/mod.rs b/crates/meilisearch-types/src/enterprise_edition/mod.rs new file mode 100644 index 000000000..47047de48 --- /dev/null +++ b/crates/meilisearch-types/src/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod network; diff --git a/crates/meilisearch-types/src/network.rs b/crates/meilisearch-types/src/enterprise_edition/network.rs similarity index 95% rename from crates/meilisearch-types/src/network.rs rename to crates/meilisearch-types/src/enterprise_edition/network.rs index 79b3b3808..9d5c51e25 100644 --- a/crates/meilisearch-types/src/network.rs +++ b/crates/meilisearch-types/src/enterprise_edition/network.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; -use milli::update::new::indexer::sharding::Shards; +use milli::update::new::indexer::enterprise_edition::sharding::Shards; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index 0cb647b0a..6c013f3bb 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -3,6 +3,7 @@ pub mod batches; pub mod compression; pub mod deserr; pub mod document_formats; +pub mod enterprise_edition; pub mod error; pub mod facet_values_sort; pub mod features; @@ -10,7 +11,6 @@ pub mod index_uid; pub mod index_uid_pattern; pub mod keys; pub mod locales; -pub mod network; pub mod settings; pub mod star_or; pub mod task_view; diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index d188a9867..d445153ad 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -45,7 +45,7 @@ use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; -use crate::routes::indexes::proxy::{proxy, Body}; +use crate::routes::indexes::enterprise_edition::proxy::{proxy, Body}; use crate::routes::indexes::search::fix_sort_query_parameters; use crate::routes::{ get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, diff --git a/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs b/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs new file mode 100644 index 000000000..a3ca7350c --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod proxy; diff --git a/crates/meilisearch/src/routes/indexes/proxy.rs b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs similarity index 98% rename from crates/meilisearch/src/routes/indexes/proxy.rs rename to crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs index a1106b974..7cb3eb8cc 100644 --- a/crates/meilisearch/src/routes/indexes/proxy.rs +++ b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs @@ -17,7 +17,9 @@ use serde::de::DeserializeOwned; use serde_json::Value; use crate::error::MeilisearchHttpError; -use crate::routes::indexes::proxy::error::{ProxyDocumentChangeError, ReqwestErrorWithoutUrl}; +use crate::routes::indexes::enterprise_edition::proxy::error::{ + ProxyDocumentChangeError, ReqwestErrorWithoutUrl, +}; use crate::routes::SummarizedTaskView; pub enum Body { @@ -50,7 +52,7 @@ pub async fn proxy( index_scheduler: &IndexScheduler, index_uid: &str, req: &HttpRequest, - network: meilisearch_types::network::Network, + network: meilisearch_types::enterprise_edition::network::Network, body: Body, task: &meilisearch_types::tasks::Task, ) -> Result<(), MeilisearchHttpError> { diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index d1946215f..8e994bb43 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -29,8 +29,8 @@ use crate::routes::is_dry_run; use crate::Opt; pub mod documents; +mod enterprise_edition; pub mod facet_search; -mod proxy; pub mod search; mod search_analytics; #[cfg(test)] @@ -40,7 +40,7 @@ mod settings_analytics; pub mod similar; mod similar_analytics; -pub use proxy::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; +pub use enterprise_edition::proxy::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; #[derive(OpenApi)] #[openapi( diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index f4119b0ad..b7fa2980c 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -7,6 +7,7 @@ use deserr::Deserr; use index_scheduler::IndexScheduler; use itertools::{EitherOrBoth, Itertools}; use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::enterprise_edition::network::{Network as DbNetwork, Remote as DbRemote}; use meilisearch_types::error::deserr_codes::{ InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkSharding, InvalidNetworkUrl, InvalidNetworkWriteApiKey, @@ -14,7 +15,6 @@ use meilisearch_types::error::deserr_codes::{ use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; -use meilisearch_types::network::{Network as DbNetwork, Remote as DbRemote}; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 6fb8bcd43..4988dc07d 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -9,12 +9,12 @@ use std::vec::{IntoIter, Vec}; use actix_http::StatusCode; use index_scheduler::{IndexScheduler, RoFeatures}; use itertools::Itertools; +use meilisearch_types::enterprise_edition::network::{Network, Remote}; use meilisearch_types::error::ResponseError; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; use meilisearch_types::milli::vector::Embedding; use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget, DEFAULT_VALUES_PER_FACET}; -use meilisearch_types::network::{Network, Remote}; use roaring::RoaringBitmap; use tokio::task::JoinHandle; diff --git a/crates/meilisearch/src/search/federated/proxy.rs b/crates/meilisearch/src/search/federated/proxy.rs index 6da92ca9e..ac8cd2491 100644 --- a/crates/meilisearch/src/search/federated/proxy.rs +++ b/crates/meilisearch/src/search/federated/proxy.rs @@ -1,6 +1,6 @@ pub use error::ProxySearchError; use error::ReqwestErrorWithoutUrl; -use meilisearch_types::network::Remote; +use meilisearch_types::enterprise_edition::network::Remote; use rand::Rng as _; use reqwest::{Client, Response, StatusCode}; use serde::de::DeserializeOwned; diff --git a/crates/milli/src/update/new/indexer/document_operation.rs b/crates/milli/src/update/new/indexer/document_operation.rs index 50364c07c..4e3725c6e 100644 --- a/crates/milli/src/update/new/indexer/document_operation.rs +++ b/crates/milli/src/update/new/indexer/document_operation.rs @@ -17,7 +17,7 @@ use super::guess_primary_key::retrieve_or_guess_primary_key; use crate::documents::PrimaryKey; use crate::progress::{AtomicPayloadStep, Progress}; use crate::update::new::document::{DocumentContext, Versions}; -use crate::update::new::indexer::sharding::Shards; +use crate::update::new::indexer::enterprise_edition::sharding::Shards; use crate::update::new::steps::IndexingStep; use crate::update::new::thread_local::MostlySend; use crate::update::new::{DocumentIdentifiers, Insertion, Update}; diff --git a/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs b/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs new file mode 100644 index 000000000..9f89e8e31 --- /dev/null +++ b/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod sharding; diff --git a/crates/milli/src/update/new/indexer/sharding.rs b/crates/milli/src/update/new/indexer/enterprise_edition/sharding.rs similarity index 100% rename from crates/milli/src/update/new/indexer/sharding.rs rename to crates/milli/src/update/new/indexer/enterprise_edition/sharding.rs diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index b24081c02..e18337623 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -31,12 +31,12 @@ pub(crate) mod de; pub mod document_changes; mod document_deletion; mod document_operation; +pub mod enterprise_edition; mod extract; mod guess_primary_key; mod partial_dump; mod post_processing; pub mod settings_changes; -pub mod sharding; mod update_by_function; mod write; From b541b7bed3f2b3e39d5fca041e39478b48e5855e Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 19 Aug 2025 14:48:31 +0200 Subject: [PATCH 38/38] Change license text to clarify that EE files are in EE modules --- LICENSE-EE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-EE b/LICENSE-EE index 09ae27573..87510480d 100644 --- a/LICENSE-EE +++ b/LICENSE-EE @@ -5,7 +5,7 @@ Parameters Licensor: Meili SAS -Licensed Work: Any file explicitly marked as “Enterprise Edition (EE)” or “governed by the Business Source License”. +Licensed Work: Any file explicitly marked as “Enterprise Edition (EE)” or “governed by the Business Source License” residing in enterprise_editions modules/folders. Additional Use Grant: You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation.