diff --git a/Cargo.lock b/Cargo.lock index c5db441f1..f4bb3a407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -589,7 +589,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "bumpalo", @@ -779,7 +779,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "time", @@ -1812,7 +1812,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "big_s", @@ -2054,7 +2054,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.17.1" +version = "1.18.0" dependencies = [ "tempfile", "thiserror 2.0.12", @@ -2076,7 +2076,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.17.1" +version = "1.18.0" dependencies = [ "insta", "levenshtein_automata", @@ -2098,7 +2098,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.17.1" +version = "1.18.0" dependencies = [ "criterion", "serde_json", @@ -2255,7 +2255,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.17.1" +version = "1.18.0" dependencies = [ "arbitrary", "bumpalo", @@ -3173,7 +3173,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "backoff", @@ -3450,7 +3450,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.17.1" +version = "1.18.0" dependencies = [ "criterion", "serde_json", @@ -3944,7 +3944,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.17.1" +version = "1.18.0" dependencies = [ "insta", "md5", @@ -3955,7 +3955,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.17.1" +version = "1.18.0" dependencies = [ "actix-cors", "actix-http", @@ -3965,6 +3965,7 @@ dependencies = [ "actix-web-lab", "anyhow", "async-openai", + "backoff", "brotli", "bstr", "build-info", @@ -4051,7 +4052,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.17.1" +version = "1.18.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -4070,7 +4071,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.17.1" +version = "1.18.0" dependencies = [ "actix-web", "anyhow", @@ -4105,7 +4106,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "clap", @@ -4139,7 +4140,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.17.1" +version = "1.18.0" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4212,6 +4213,7 @@ dependencies = [ "time", "tokenizers", "tracing", + "twox-hash", "ureq", "url", "utoipa", @@ -4707,7 +4709,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.17.1" +version = "1.18.0" dependencies = [ "big_s", "serde_json", @@ -6717,6 +6719,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" + [[package]] name = "typeid" version = "1.0.3" @@ -7546,7 +7554,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.17.1" +version = "1.18.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index bc1c354b7..d68fa222c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "1.17.1" +version = "1.18.0" authors = [ "Quentin de Quelen ", "Clément Renault ", diff --git a/LICENSE b/LICENSE index 686f24931..c6ed62f37 100644 --- a/LICENSE +++ b/LICENSE @@ -19,3 +19,11 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +🔒 Meilisearch Enterprise Edition (EE) + +Certain parts of this codebase are not licensed under the MIT license and governed by the Business Source License 1.1. + +See the LICENSE-EE file for details. diff --git a/LICENSE-EE b/LICENSE-EE new file mode 100644 index 000000000..87510480d --- /dev/null +++ b/LICENSE-EE @@ -0,0 +1,67 @@ +Business Source License 1.1 – Adapted for Meili SAS +This license is based on the Business Source License version 1.1, as published by MariaDB Corporation Ab. + +Parameters + +Licensor: Meili SAS + +Licensed Work: Any file explicitly marked as “Enterprise Edition (EE)” or “governed by the Business Source License” residing in enterprise_editions modules/folders. + +Additional Use Grant: +You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation. + +Production use of the Licensed Work requires a commercial license agreement with Meilisearch. Contact bonjour@meilisearch.com for licensing. + +Change License: MIT + +Change Date: Four years from the date the Licensed Work is published. + +This License does not apply to any code outside of the Licensed Work, which remains under the MIT license. + +For information about alternative licensing arrangements for the Licensed Work, +please contact bonjour@meilisearch.com or sales@meilisearch.com. + +Notice + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. diff --git a/README.md b/README.md index 40833a0d6..aa371d650 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,26 @@ We also offer a wide range of dedicated guides to all Meilisearch features, such Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced). +## 🧾 Editions & Licensing + +Meilisearch is available in two editions: + +### 🧪 Community Edition (CE) + +- Fully open source under the [MIT license](./LICENSE) +- Core search engine with fast and relevant full-text, semantic or hybrid search +- Free to use for anyone, including commercial usage + +### 🏢 Enterprise Edition (EE) + +- Includes advanced features such as: + - Sharding +- Governed by a [commercial license](./LICENSE-EE) or the [Business Source License 1.1](https://mariadb.com/bsl11) +- Not allowed in production without a commercial agreement with Meilisearch. + - You may use, modify, and distribute the Licensed Work for non-production purposes only, such as testing, development, or evaluation. + +Want access to Enterprise features? → Contact us at [sales@meilisearch.com](maito:sales@meilisearch.com). + ## 📊 Telemetry Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want. diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 4083b69dd..75805e28c 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -154,6 +154,7 @@ fn indexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -221,6 +222,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -266,6 +268,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -335,6 +338,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -412,6 +416,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -457,6 +462,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -498,6 +504,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -566,6 +573,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -633,6 +641,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -700,6 +709,7 @@ fn indexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -766,6 +776,7 @@ fn reindexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -811,6 +822,7 @@ fn reindexing_wiki(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -879,6 +891,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -956,6 +969,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1002,6 +1016,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1044,6 +1059,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1111,6 +1127,7 @@ fn indexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1177,6 +1194,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1222,6 +1240,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1290,6 +1309,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1404,6 +1424,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1449,6 +1470,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1490,6 +1512,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1580,6 +1603,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1671,6 +1695,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1754,6 +1779,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1821,6 +1847,7 @@ fn indexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1887,6 +1914,7 @@ fn reindexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -1932,6 +1960,7 @@ fn reindexing_geo(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2000,6 +2029,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 0abbd6c71..44066767a 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -123,6 +123,7 @@ pub fn base_setup(conf: &Conf) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index 81ba40944..a2b72e0e5 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -10,7 +10,7 @@ use meilisearch_types::keys::Key; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::settings::Unchecked; use meilisearch_types::tasks::{ - Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId, + Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId, TaskNetwork, }; use meilisearch_types::InstanceUid; use roaring::RoaringBitmap; @@ -94,6 +94,8 @@ pub struct TaskDump { default )] pub finished_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } // A `Kind` specific version made for the dump. If modified you may break the dump. @@ -129,6 +131,7 @@ pub enum KindDump { }, IndexUpdate { primary_key: Option, + uid: Option, }, IndexSwap { swaps: Vec, @@ -171,6 +174,7 @@ impl From for TaskDump { enqueued_at: task.enqueued_at, started_at: task.started_at, finished_at: task.finished_at, + network: task.network, } } } @@ -210,8 +214,8 @@ impl From for KindDump { KindWithContent::IndexCreation { primary_key, .. } => { KindDump::IndexCreation { primary_key } } - KindWithContent::IndexUpdate { primary_key, .. } => { - KindDump::IndexUpdate { primary_key } + KindWithContent::IndexUpdate { primary_key, new_index_uid: uid, .. } => { + KindDump::IndexUpdate { primary_key, uid } } KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps }, KindWithContent::TaskCancelation { query, tasks } => { @@ -249,8 +253,9 @@ pub(crate) mod test { use big_s::S; use maplit::{btreemap, btreeset}; use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats}; + use meilisearch_types::enterprise_edition::network::{Network, Remote}; use meilisearch_types::facet_values_sort::FacetValuesSort; - use meilisearch_types::features::{Network, Remote, RuntimeTogglableFeatures}; + use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::keys::{Action, Key}; use meilisearch_types::milli::update::Setting; @@ -383,6 +388,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-11 0:00 UTC), started_at: Some(datetime!(2022-11-20 0:00 UTC)), finished_at: Some(datetime!(2022-11-21 0:00 UTC)), + network: None, }, None, ), @@ -407,6 +413,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-11 0:00 UTC), started_at: None, finished_at: None, + network: None, }, Some(vec![ json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(), @@ -426,6 +433,7 @@ pub(crate) mod test { enqueued_at: datetime!(2022-11-15 0:00 UTC), started_at: None, finished_at: None, + network: None, }, None, ), @@ -538,7 +546,8 @@ pub(crate) mod test { fn create_test_network() -> Network { Network { local: Some("myself".to_string()), - remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()) }}, + remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }}, + sharding: false, } } diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 3a0c8ef0d..cdfa6847e 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -85,7 +85,7 @@ impl CompatV5ToV6 { v6::Kind::IndexCreation { primary_key } } v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => { - v6::Kind::IndexUpdate { primary_key } + v6::Kind::IndexUpdate { primary_key, uid: None } } v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion, v5::tasks::TaskContent::DocumentAddition { @@ -140,9 +140,11 @@ impl CompatV5ToV6 { v5::Details::Settings { settings } => { v6::Details::SettingsUpdate { settings: Box::new(settings.into()) } } - v5::Details::IndexInfo { primary_key } => { - v6::Details::IndexInfo { primary_key } - } + v5::Details::IndexInfo { primary_key } => v6::Details::IndexInfo { + primary_key, + new_index_uid: None, + old_index_uid: None, + }, v5::Details::DocumentDeletion { received_document_ids, deleted_documents, @@ -161,6 +163,7 @@ impl CompatV5ToV6 { enqueued_at: task_view.enqueued_at, started_at: task_view.started_at, finished_at: task_view.finished_at, + network: None, }; (task, content_file) diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 9bc4b33c5..75ff2ebe6 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch; pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; -pub type Network = meilisearch_types::features::Network; +pub type Network = meilisearch_types::enterprise_edition::network::Network; pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 1d41b6aa5..1f8000a50 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -5,7 +5,8 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; -use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; +use meilisearch_types::enterprise_edition::network::Network; +use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; use meilisearch_types::webhooks::WebhooksDumpView; diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rs index 39ed9482b..d43868c67 100644 --- a/crates/file-store/src/lib.rs +++ b/crates/file-store/src/lib.rs @@ -148,11 +148,10 @@ impl File { Ok(Self { path: PathBuf::new(), file: None }) } - pub fn persist(self) -> Result<()> { - if let Some(file) = self.file { - file.persist(&self.path)?; - } - Ok(()) + pub fn persist(self) -> Result> { + let Some(file) = self.file else { return Ok(None) }; + + Ok(Some(file.persist(&self.path)?)) } } diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index ec1f96fd5..7ca4d3092 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -129,6 +129,7 @@ fn main() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs index 1e681c8e8..e0866036b 100644 --- a/crates/index-scheduler/src/dump.rs +++ b/crates/index-scheduler/src/dump.rs @@ -147,6 +147,7 @@ impl<'a> Dump<'a> { canceled_by: task.canceled_by, details: task.details, status: task.status, + network: task.network, kind: match task.kind { KindDump::DocumentImport { primary_key, @@ -197,9 +198,10 @@ impl<'a> Dump<'a> { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, }, - KindDump::IndexUpdate { primary_key } => KindWithContent::IndexUpdate { + KindDump::IndexUpdate { primary_key, uid } => KindWithContent::IndexUpdate { index_uid: task.index_uid.ok_or(Error::CorruptedDump)?, primary_key, + new_index_uid: uid, }, KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps }, KindDump::TaskCancelation { query, tasks } => { diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 60669ff2d..332b7e040 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -67,6 +67,8 @@ pub enum Error { SwapDuplicateIndexesFound(Vec), #[error("Index `{0}` not found.")] SwapIndexNotFound(String), + #[error("Cannot rename `{0}` to `{1}` as the index already exists. Hint: You can remove `{1}` first and then do your remove.")] + SwapIndexFoundDuringRename(String, String), #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")] NoSpaceLeftInTaskQueue, #[error( @@ -74,6 +76,10 @@ pub enum Error { .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") )] SwapIndexesNotFound(Vec), + #[error("The following indexes are being renamed but cannot because their new name conflicts with an already existing index: {}. Renaming doesn't overwrite the other index name.", + .0.iter().map(|s| format!("`{}`", s)).collect::>().join(", ") + )] + SwapIndexesFoundDuringRename(Vec), #[error("Corrupted dump.")] CorruptedDump, #[error( @@ -203,6 +209,8 @@ impl Error { | Error::SwapIndexNotFound(_) | Error::NoSpaceLeftInTaskQueue | Error::SwapIndexesNotFound(_) + | Error::SwapIndexFoundDuringRename(_, _) + | Error::SwapIndexesFoundDuringRename(_) | Error::CorruptedDump | Error::InvalidTaskDate { .. } | Error::InvalidTaskUid { .. } @@ -271,6 +279,8 @@ impl ErrorCode for Error { Error::SwapDuplicateIndexFound(_) => Code::InvalidSwapDuplicateIndexFound, Error::SwapIndexNotFound(_) => Code::IndexNotFound, Error::SwapIndexesNotFound(_) => Code::IndexNotFound, + Error::SwapIndexFoundDuringRename(_, _) => Code::IndexAlreadyExists, + Error::SwapIndexesFoundDuringRename(_) => Code::IndexAlreadyExists, Error::InvalidTaskDate { field, .. } => (*field).into(), Error::InvalidTaskUid { .. } => Code::InvalidTaskUids, Error::InvalidBatchUid { .. } => Code::InvalidBatchUids, diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index dee665458..1b01f89de 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -1,6 +1,7 @@ use std::sync::{Arc, RwLock}; -use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures}; +use meilisearch_types::enterprise_edition::network::Network; +use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{SerdeJson, Str}; use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls}; diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index e6bdccd41..d578b03dd 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -526,6 +526,20 @@ impl IndexMapper { Ok(()) } + /// Rename an index. + pub fn rename(&self, wtxn: &mut RwTxn, current: &str, new: &str) -> Result<()> { + let uuid = self + .index_mapping + .get(wtxn, current)? + .ok_or_else(|| Error::IndexNotFound(current.to_string()))?; + if self.index_mapping.get(wtxn, new)?.is_some() { + return Err(Error::IndexAlreadyExists(new.to_string())); + } + self.index_mapping.delete(wtxn, current)?; + self.index_mapping.put(wtxn, new, &uuid)?; + Ok(()) + } + /// The stats of an index. /// /// If available in the cache, they are directly returned. diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index cb804d9b4..df043ad87 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -230,6 +230,7 @@ pub fn snapshot_task(task: &Task) -> String { details, status, kind, + network, } = task; snap.push('{'); snap.push_str(&format!("uid: {uid}, ")); @@ -247,6 +248,9 @@ pub fn snapshot_task(task: &Task) -> String { snap.push_str(&format!("details: {}, ", &snapshot_details(details))); } snap.push_str(&format!("kind: {kind:?}")); + if let Some(network) = network { + snap.push_str(&format!("network: {network:?}, ")) + } snap.push('}'); snap @@ -274,8 +278,8 @@ fn snapshot_details(d: &Details) -> String { Details::SettingsUpdate { settings } => { format!("{{ settings: {settings:?} }}") } - Details::IndexInfo { primary_key } => { - format!("{{ primary_key: {primary_key:?} }}") + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => { + format!("{{ primary_key: {primary_key:?}, old_new_uid: {old_index_uid:?}, new_index_uid: {new_index_uid:?} }}") } Details::DocumentDeletion { provided_ids: received_document_ids, diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 6ad7a8397..d6c150dbb 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -51,8 +51,9 @@ pub use features::RoFeatures; use flate2::bufread::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; +use meilisearch_types::enterprise_edition::network::Network; use meilisearch_types::features::{ - ChatCompletionSettings, InstanceTogglableFeatures, Network, RuntimeTogglableFeatures, + ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures, }; use meilisearch_types::heed::byteorder::BE; use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128}; @@ -64,7 +65,7 @@ use meilisearch_types::milli::vector::{ }; use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; -use meilisearch_types::tasks::{KindWithContent, Task}; +use meilisearch_types::tasks::{KindWithContent, Task, TaskNetwork}; use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; @@ -666,6 +667,16 @@ impl IndexScheduler { self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, &processing) } + pub fn set_task_network(&self, task_id: TaskId, network: TaskNetwork) -> Result<()> { + let mut wtxn = self.env.write_txn()?; + let mut task = + self.queue.tasks.get_task(&wtxn, task_id)?.ok_or(Error::TaskNotFound(task_id))?; + task.network = Some(network); + self.queue.tasks.all_tasks.put(&mut wtxn, &task_id, &task)?; + wtxn.commit()?; + Ok(()) + } + /// Return the batches matching the query from the user's point of view along /// with the total number of batches matching the query, ignoring from and limit. /// diff --git a/crates/index-scheduler/src/queue/batches_test.rs b/crates/index-scheduler/src/queue/batches_test.rs index 782acb4b1..49dc0ba61 100644 --- a/crates/index-scheduler/src/queue/batches_test.rs +++ b/crates/index-scheduler/src/queue/batches_test.rs @@ -334,11 +334,11 @@ fn query_batches_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -442,7 +442,7 @@ fn query_batches_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/queue/mod.rs b/crates/index-scheduler/src/queue/mod.rs index 92de10fe1..4f02edfb2 100644 --- a/crates/index-scheduler/src/queue/mod.rs +++ b/crates/index-scheduler/src/queue/mod.rs @@ -279,6 +279,7 @@ impl Queue { details: kind.default_details(), status: Status::Enqueued, kind: kind.clone(), + network: None, }; // For deletion and cancelation tasks, we want to make extra sure that they // don't attempt to delete/cancel tasks that are newer than themselves. diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap index 48d1ccaab..ec05be8f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap index 4c54de49a..a30869296 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap index 74c4c4a33..9a335acf5 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap index 411e82ea0..022243e3b 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap index 4c76db95e..08aa4856e 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap index 7ce0d3ca3..fcd523744 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/after-advancing-a-bit.snap @@ -7,9 +7,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap index 603544991..b3443a0ad 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap index 6dc897dfa..2039314cf 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap index 84d6c7878..1e9eaa775 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/after-processing-everything.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/queue/batches_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `whalo` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -54,8 +54,8 @@ doggo: { number_of_documents: 0, field_distribution: {} } ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } 1 {uid: 1, details: {"primaryKey":"sheep"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } -2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } -3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } +2 {uid: 2, details: {"swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 2 of type `indexSwap` that cannot be batched with any other task.", } +3 {uid: 3, details: {"swaps":[{"indexes":["catto","whalo"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 3 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap index 1184c197f..26271c168 100644 --- a/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/batches_test.rs/query_batches_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/batches_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap index e3c26b2b3..c10efb84a 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_canceled_by/start.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: canceled, canceled_by: 3, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 1, status: canceled, canceled_by: 3, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { matched_tasks: 3, canceled_tasks: Some(2), original_filter: "test_query" }, kind: TaskCancelation { query: "test_query", tasks: RoaringBitmap<[0, 1, 2]> }} ---------------------------------------------------------------------- ### Status: @@ -49,7 +49,7 @@ catto: { number_of_documents: 0, field_distribution: {} } ---------------------------------------------------------------------- ### All Batches: 0 {uid: 0, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } -1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"]}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } +1 {uid: 1, details: {"primaryKey":"sheep","matchedTasks":3,"canceledTasks":2,"originalFilter":"test_query","swaps":[{"indexes":["catto","doggo"],"rename":false}]}, stats: {"totalNbTasks":3,"status":{"succeeded":1,"canceled":2},"types":{"indexCreation":1,"indexSwap":1,"taskCancelation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `taskCancelation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap index 4475c71fc..af00b64d0 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/processed_all_tasks.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap index 4f9ffb209..b56d82bcd 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap index eb6b0e7ec..d5e66a7de 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap index 181f0308c..3e93b86b9 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_from_and_limit/registered_the_third_task.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("plankton"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("plankton") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("his_own_vomit"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("his_own_vomit") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap index 4ac6201a6..939f8e9f6 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/end.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/queue/tasks_test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap index 268f463aa..80b1d71be 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_simple/start.snap @@ -1,15 +1,14 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("fish") }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("fish"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "whalo", primary_key: Some("fish") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,] diff --git a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap index 60c041c05..3949d7c90 100644 --- a/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap +++ b/crates/index-scheduler/src/queue/snapshots/tasks_test.rs/query_tasks_special_rules/start.snap @@ -1,16 +1,15 @@ --- source: crates/index-scheduler/src/queue/tasks_test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} -2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo") }] }} -3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo") }] }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("sheep"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("sheep") }} +2 {uid: 2, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "doggo"), rename: false }] }} +3 {uid: 3, status: enqueued, details: { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("catto", "whalo"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,2,3,] diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap index e4d9af541..ae4ff5b21 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/register/everything_is_successfully_registered.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 12, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 12, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 50, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 50, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 5000, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 5000, allow_index_creation: true }} diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap index 30e8e17a8..51dee86b1 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/after_the_second_task_deletion.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap index 4e3fb5439..96d6c0c9a 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -39,7 +40,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -60,7 +63,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -81,7 +86,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap index 4cabce94b..99e57fec2 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_deletion_have_been_processed.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -34,7 +35,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -39,7 +40,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -60,7 +63,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -81,7 +86,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap index 5565994cb..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -39,7 +40,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -60,7 +63,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -81,7 +86,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap index 5565994cb..3cc7f85a0 100644 --- a/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap +++ b/crates/index-scheduler/src/queue/snapshots/test.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap @@ -1,6 +1,5 @@ --- source: crates/index-scheduler/src/queue/test.rs -snapshot_kind: text --- [ { @@ -13,7 +12,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "succeeded", @@ -39,7 +40,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "failed", @@ -60,7 +63,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", @@ -81,7 +86,9 @@ snapshot_kind: text "canceledBy": null, "details": { "IndexInfo": { - "primary_key": null + "primary_key": null, + "new_index_uid": null, + "old_index_uid": null } }, "status": "enqueued", diff --git a/crates/index-scheduler/src/queue/tasks.rs b/crates/index-scheduler/src/queue/tasks.rs index 74192232e..83c698ebe 100644 --- a/crates/index-scheduler/src/queue/tasks.rs +++ b/crates/index-scheduler/src/queue/tasks.rs @@ -97,7 +97,22 @@ impl TaskQueue { Ok(self.all_tasks.get(rtxn, &task_id)?) } - pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &Task) -> Result<()> { + /// Update the inverted task indexes and write the new value of the task. + /// + /// The passed `task` object typically comes from a previous transaction, so two kinds of modification might have occurred: + /// 1. Modification to the `task` object after loading it from the DB (the purpose of this method is to persist these changes) + /// 2. Modification to the task committed by another transaction in the DB (an annoying consequence of having lost the original + /// transaction from which the `task` instance was deserialized) + /// + /// When calling this function, this `task` is modified to take into account any existing `network` + /// that can have been added since the task was loaded into memory. + /// + /// Any other modification to the task that was committed from the DB since the parameter was pulled from the DB will be overwritten. + /// + /// # Errors + /// + /// - CorruptedTaskQueue: The task doesn't exist in the database + pub(crate) fn update_task(&self, wtxn: &mut RwTxn, task: &mut Task) -> Result<()> { let old_task = self.get_task(wtxn, task.uid)?.ok_or(Error::CorruptedTaskQueue)?; let reprocessing = old_task.status != Status::Enqueued; @@ -157,6 +172,12 @@ impl TaskQueue { } } + task.network = match (old_task.network, task.network.take()) { + (None, None) => None, + (None, Some(network)) | (Some(network), None) => Some(network), + (Some(_), Some(network)) => Some(network), + }; + self.all_tasks.put(wtxn, &task.uid, task)?; Ok(()) } diff --git a/crates/index-scheduler/src/queue/tasks_test.rs b/crates/index-scheduler/src/queue/tasks_test.rs index d60d621d1..bfe26accf 100644 --- a/crates/index-scheduler/src/queue/tasks_test.rs +++ b/crates/index-scheduler/src/queue/tasks_test.rs @@ -304,11 +304,11 @@ fn query_tasks_special_rules() { let kind = index_creation_task("doggo", "sheep"); let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "whalo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); @@ -399,7 +399,7 @@ fn query_tasks_canceled_by() { let kind = index_creation_task("doggo", "sheep"); let _ = index_scheduler.register(kind, None, false).unwrap(); let kind = KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("catto".to_owned(), "doggo".to_owned()), rename: false }], }; let _task = index_scheduler.register(kind, None, false).unwrap(); diff --git a/crates/index-scheduler/src/scheduler/autobatcher.rs b/crates/index-scheduler/src/scheduler/autobatcher.rs index b3f7d2743..a88a9f0bf 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher.rs @@ -287,7 +287,7 @@ impl BatchKind { }; match (self, autobatch_kind) { - // We don't batch any of these operations + // We don't batch any of these operations (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break((this, BatchStopReason::TaskCannotBeBatched { kind, id })), // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists. (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => { diff --git a/crates/index-scheduler/src/scheduler/autobatcher_test.rs b/crates/index-scheduler/src/scheduler/autobatcher_test.rs index 435ce55c6..7dd4536b7 100644 --- a/crates/index-scheduler/src/scheduler/autobatcher_test.rs +++ b/crates/index-scheduler/src/scheduler/autobatcher_test.rs @@ -75,7 +75,11 @@ fn idx_create() -> KindWithContent { } fn idx_update() -> KindWithContent { - KindWithContent::IndexUpdate { index_uid: String::from("doggo"), primary_key: None } + KindWithContent::IndexUpdate { + index_uid: String::from("doggo"), + primary_key: None, + new_index_uid: None, + } } fn idx_del() -> KindWithContent { @@ -84,7 +88,10 @@ fn idx_del() -> KindWithContent { fn idx_swap() -> KindWithContent { KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: (String::from("doggo"), String::from("catto")) }], + swaps: vec![IndexSwap { + indexes: (String::from("doggo"), String::from("catto")), + rename: false, + }], } } diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index e78ed2c2e..14aa307cd 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -38,6 +38,7 @@ pub(crate) enum Batch { IndexUpdate { index_uid: String, primary_key: Option, + new_index_uid: Option, task: Task, }, IndexDeletion { @@ -405,11 +406,13 @@ impl IndexScheduler { let mut task = self.queue.tasks.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?; current_batch.processing(Some(&mut task)); - let primary_key = match &task.kind { - KindWithContent::IndexUpdate { primary_key, .. } => primary_key.clone(), + let (primary_key, new_index_uid) = match &task.kind { + KindWithContent::IndexUpdate { primary_key, new_index_uid, .. } => { + (primary_key.clone(), new_index_uid.clone()) + } _ => unreachable!(), }; - Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task })) + Ok(Some(Batch::IndexUpdate { index_uid, primary_key, new_index_uid, task })) } BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion { index_uid, diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index b2bb90c0b..c57bbf70d 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -268,7 +268,7 @@ impl IndexScheduler { self.queue .tasks - .update_task(&mut wtxn, &task) + .update_task(&mut wtxn, &mut task) .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } if let Some(canceled_by) = canceled_by { @@ -349,7 +349,7 @@ impl IndexScheduler { self.queue .tasks - .update_task(&mut wtxn, &task) + .update_task(&mut wtxn, &mut task) .map_err(|e| Error::UnrecoverableError(Box::new(e)))?; } } diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index c21ab27ad..4129c57af 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -10,6 +10,7 @@ use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use milli::update::Settings as MilliSettings; use roaring::RoaringBitmap; +use time::OffsetDateTime; use super::create_batch::Batch; use crate::processing::{ @@ -224,24 +225,46 @@ impl IndexScheduler { self.index_mapper.create_index(wtxn, &index_uid, None)?; self.process_batch( - Batch::IndexUpdate { index_uid, primary_key, task }, + Batch::IndexUpdate { index_uid, primary_key, new_index_uid: None, task }, current_batch, progress, ) } - Batch::IndexUpdate { index_uid, primary_key, mut task } => { + Batch::IndexUpdate { index_uid, primary_key, new_index_uid, mut task } => { progress.update_progress(UpdateIndexProgress::UpdatingTheIndex); + + // Get the index (renamed or not) let rtxn = self.env.read_txn()?; let index = self.index_mapper.index(&rtxn, &index_uid)?; + let mut index_wtxn = index.write_txn()?; - if let Some(primary_key) = primary_key.clone() { - let mut index_wtxn = index.write_txn()?; + // Handle rename if new_index_uid is provided + let final_index_uid = if let Some(new_uid) = &new_index_uid { + if new_uid != &index_uid { + index.set_updated_at(&mut index_wtxn, &OffsetDateTime::now_utc())?; + + let mut wtxn = self.env.write_txn()?; + self.apply_index_swap( + &mut wtxn, &progress, task.uid, &index_uid, new_uid, true, + )?; + wtxn.commit()?; + + new_uid.clone() + } else { + new_uid.clone() + } + } else { + index_uid.clone() + }; + + // Handle primary key update if provided + if let Some(ref primary_key) = primary_key { let mut builder = MilliSettings::new( &mut index_wtxn, &index, self.index_mapper.indexer_config(), ); - builder.set_primary_key(primary_key); + builder.set_primary_key(primary_key.clone()); let must_stop_processing = self.scheduler.must_stop_processing.clone(); builder @@ -250,15 +273,20 @@ impl IndexScheduler { &progress, current_batch.embedder_stats.clone(), ) - .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?; - index_wtxn.commit()?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.to_string())))?; } + index_wtxn.commit()?; // drop rtxn before starting a new wtxn on the same db rtxn.commit()?; task.status = Status::Succeeded; - task.details = Some(Details::IndexInfo { primary_key }); + task.details = Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_index_uid: new_index_uid.clone(), + // we only display the old index uid if a rename happened => there is a new_index_uid + old_index_uid: new_index_uid.map(|_| index_uid.clone()), + }); // if the update processed successfully, we're going to store the new // stats of the index. Since the tasks have already been processed and @@ -268,8 +296,8 @@ impl IndexScheduler { let mut wtxn = self.env.write_txn()?; let index_rtxn = index.read_txn()?; let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn) - .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; - self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?; + .map_err(|e| Error::from_milli(e, Some(final_index_uid.clone())))?; + self.index_mapper.store_stats_of(&mut wtxn, &final_index_uid, &stats)?; wtxn.commit()?; Ok(()) }(); @@ -330,13 +358,18 @@ impl IndexScheduler { unreachable!() }; let mut not_found_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { - for index in [lhs, rhs] { - let index_exists = self.index_mapper.index_exists(&wtxn, index)?; - if !index_exists { - not_found_indexes.insert(index); - } + let mut found_indexes_but_should_not = BTreeSet::new(); + for IndexSwap { indexes: (lhs, rhs), rename } in swaps { + let index_exists = self.index_mapper.index_exists(&wtxn, lhs)?; + if !index_exists { + not_found_indexes.insert(lhs); } + let index_exists = self.index_mapper.index_exists(&wtxn, rhs)?; + match (index_exists, rename) { + (true, true) => found_indexes_but_should_not.insert((lhs, rhs)), + (false, false) => not_found_indexes.insert(rhs), + (true, false) | (false, true) => true, // random value we don't read it anyway + }; } if !not_found_indexes.is_empty() { if not_found_indexes.len() == 1 { @@ -349,6 +382,23 @@ impl IndexScheduler { )); } } + if !found_indexes_but_should_not.is_empty() { + if found_indexes_but_should_not.len() == 1 { + let (lhs, rhs) = found_indexes_but_should_not + .into_iter() + .next() + .map(|(lhs, rhs)| (lhs.clone(), rhs.clone())) + .unwrap(); + return Err(Error::SwapIndexFoundDuringRename(lhs, rhs)); + } else { + return Err(Error::SwapIndexesFoundDuringRename( + found_indexes_but_should_not + .into_iter() + .map(|(_, rhs)| rhs.to_string()) + .collect(), + )); + } + } progress.update_progress(SwappingTheIndexes::SwappingTheIndexes); for (step, swap) in swaps.iter().enumerate() { progress.update_progress(VariableNameStep::::new( @@ -362,6 +412,7 @@ impl IndexScheduler { task.uid, &swap.indexes.0, &swap.indexes.1, + swap.rename, )?; } wtxn.commit()?; @@ -451,6 +502,7 @@ impl IndexScheduler { task_id: u32, lhs: &str, rhs: &str, + rename: bool, ) -> Result<()> { progress.update_progress(InnerSwappingTwoIndexes::RetrieveTheTasks); // 1. Verify that both lhs and rhs are existing indexes @@ -458,16 +510,23 @@ impl IndexScheduler { if !index_lhs_exists { return Err(Error::IndexNotFound(lhs.to_owned())); } - let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; - if !index_rhs_exists { - return Err(Error::IndexNotFound(rhs.to_owned())); + if !rename { + let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?; + if !index_rhs_exists { + return Err(Error::IndexNotFound(rhs.to_owned())); + } } // 2. Get the task set for index = name that appeared before the index swap task let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?; index_lhs_task_ids.remove_range(task_id..); - let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; - index_rhs_task_ids.remove_range(task_id..); + let index_rhs_task_ids = if rename { + let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?; + index_rhs_task_ids.remove_range(task_id..); + index_rhs_task_ids + } else { + RoaringBitmap::new() + }; // 3. before_name -> new_name in the task's KindWithContent progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks); @@ -496,7 +555,11 @@ impl IndexScheduler { })?; // 6. Swap in the index mapper - self.index_mapper.swap(wtxn, lhs, rhs)?; + if rename { + self.index_mapper.rename(wtxn, lhs, rhs)?; + } else { + self.index_mapper.swap(wtxn, lhs, rhs)?; + } Ok(()) } diff --git a/crates/index-scheduler/src/scheduler/process_index_operation.rs b/crates/index-scheduler/src/scheduler/process_index_operation.rs index 62d0e6545..56e2d4693 100644 --- a/crates/index-scheduler/src/scheduler/process_index_operation.rs +++ b/crates/index-scheduler/src/scheduler/process_index_operation.rs @@ -66,6 +66,11 @@ impl IndexScheduler { } IndexOperation::DocumentOperation { index_uid, primary_key, operations, mut tasks } => { progress.update_progress(DocumentOperationProgress::RetrievingConfig); + + let network = self.network(); + + let shards = network.shards(); + // TODO: at some point, for better efficiency we might want to reuse the bumpalo for successive batches. // this is made difficult by the fact we're doing private clones of the index scheduler and sending it // to a fresh thread. @@ -130,6 +135,7 @@ impl IndexScheduler { &mut new_fields_ids_map, &|| must_stop_processing.get(), progress.clone(), + shards.as_ref(), ) .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?; diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap index ed6e75a3d..e19a94620 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafos", primary_key: None }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 4 {uid: 4, batch_uid: 4, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "cattos" }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "girafos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap index f98e5d308..ae975d843 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/before_index_creation.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap index ae1139c0c..298ae423f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/both_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(0) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap index 92f24508c..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap index 5f25c2964..0fe2d8915 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap index 0006ee8c0..50fb4b954 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/document_addition_and_index_deletion/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap index 17b69061a..7341d77bc 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/after_batch_creation.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap index cf2b2b691..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap index c8a407554..aebb02551 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_second_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap index 0cae69a70..5eb5d37c6 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/insert_task_while_another_task_is_processing/registered_the_third_task.snap @@ -7,8 +7,8 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"indexCreation":1},"indexUids":{"index_a":1}}, stop reason: "created batch containing only task with id 0 of type `indexCreation` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_b", primary_key: Some("id") }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "index_a" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap index c5e3e66c8..5aeff9852 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_first_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap index 8da1b6ca8..234bdc6a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_second_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap index 8ee0bfcef..1b4f9078c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/processed_the_third_task.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap index 92f24508c..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap index 21a6a59f7..555ac8fed 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_second_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap index adf9a76fe..d259336c0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_inserted_without_new_signal/registered_the_third_task.snap @@ -1,14 +1,13 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} -1 {uid: 1, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +1 {uid: 1, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "cattos", primary_key: None }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: IndexDeletion { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap index a7215f32c..0e5f2a994 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/first.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap index 1c14b091f..0643dec6b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/fourth.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap index 5c8082f72..9604b01ad 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap index a22004697..ae611686b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_fourth_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap index 635491dc1..7b995cf10 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_second_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap index 1d190baca..9ac27f0e7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/registered_the_third_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap index da91440ab..e6f48e5fe 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/second.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap index 95bc1f7f5..8759072a4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/process_tasks_without_autobatching/third.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { deleted_documents: Some(0) }, kind: DocumentClear { index_uid: "doggos" }} 3 {uid: 3, status: enqueued, details: { deleted_documents: None }, kind: DocumentClear { index_uid: "doggos" }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap index 4878bbe28..37afc3447 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_a.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap index 5a851f373..683d60cef 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_b.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [2,3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap index dad7609d2..d2f999560 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_c.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [3,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap index ee0a12692..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/create_d.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap index 39d1b3339..e7b05d39e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [5,] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [1,4,5,] -b [0,4,] -c [3,4,5,] -d [2,4,] +a [4,5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -64,7 +64,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap index 5d292fe21..ce45dfe50 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_registered.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap index 9327015c4..0c23e839b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,] "indexSwap" [4,5,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -66,8 +66,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap index f85735397..3c9dbc998 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b") }, IndexSwap { indexes: ("a", "d") }] }} -5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }} +5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }} ---------------------------------------------------------------------- ### Status: @@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,] "indexSwap" [4,5,6,] ---------------------------------------------------------------------- ### Index Tasks: -a [3,4,5,] -b [0,4,] -c [1,4,5,] -d [2,4,] +a [5,] +b [0,1,4,] +c [4,5,] +d [2,3,4,] ---------------------------------------------------------------------- ### Index Mapper: a: { number_of_documents: 0, field_distribution: {} } @@ -70,8 +70,8 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","d"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } -5 {uid: 5, details: {"swaps":[{"indexes":["a","c"]}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","d"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +5 {uid: 5, details: {"swaps":[{"indexes":["a","c"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 5 of type `indexSwap` that cannot be batched with any other task.", } 6 {uid: 6, details: {"swaps":[]}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 6 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap index 46d70dceb..c79a43013 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/two_swaps_registered.snap @@ -6,12 +6,12 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "d") }] }} -5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }} +5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [4,5,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap index ee0a12692..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/after_the_index_creation.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap index da9340f3b..604db5003 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/first_swap_failed.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} -4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b") }, IndexSwap { indexes: ("c", "e") }, IndexSwap { indexes: ("d", "f") }] }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +4 {uid: 4, batch_uid: 4, status: failed, error: ResponseError { code: 200, message: "Indexes `e`, `f` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "e"), rename: false }, IndexSwap { indexes: ("d", "f"), rename: false }] }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -65,7 +65,7 @@ d: { number_of_documents: 0, field_distribution: {} } 1 {uid: 1, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"b":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"c":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"id"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"d":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } -4 {uid: 4, details: {"swaps":[{"indexes":["a","b"]},{"indexes":["c","e"]},{"indexes":["d","f"]}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } +4 {uid: 4, details: {"swaps":[{"indexes":["a","b"],"rename":false},{"indexes":["c","e"],"rename":false},{"indexes":["d","f"],"rename":false}]}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexSwap":1},"indexUids":{}}, stop reason: "created batch containing only task with id 4 of type `indexSwap` that cannot be batched with any other task.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap index ee0a12692..8340e5647 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes_errors/initial_tasks_processed.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} -3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }} +3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap index 46cbaefc2..b8979eb88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/initial_tasks_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} ---------------------------------------------------------------------- diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap index 3b89fe1e7..daaeb92be 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_done.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 0, status: succeeded, details: { matched_tasks: 2, deleted_tasks: Some(0), original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap index a861fea12..fe9332a88 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_enqueued.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap index d8abc1314..9ab62e8d7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/task_deletion_undeleteable/task_deletion_processing.snap @@ -7,7 +7,7 @@ source: crates/index-scheduler/src/scheduler/test.rs {uid: 0, details: {"matchedTasks":2,"deletedTasks":null,"originalFilter":"test_query"}, stats: {"totalNbTasks":1,"status":{"processing":1},"types":{"taskDeletion":1},"indexUids":{}}, stop reason: "stopped after the last task of type `taskDeletion` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "catto", primary_key: None, method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggo", primary_key: Some("bone"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { matched_tasks: 2, deleted_tasks: None, original_filter: "test_query" }, kind: TaskDeletion { query: "test_query", tasks: RoaringBitmap<[0, 1]> }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap index 1dde1a394..c2d6d5f07 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/after_restart.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap index dd1d76f55..b4c0285c1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_scheduler_doesnt_run_with_zero_batched_tasks/registered_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap index cf2b2b691..bfed3f217 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test.rs/test_task_is_processing/registered_a_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("id") }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "index_a", primary_key: Some("id") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap index f8caaa995..8cc9bd96b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_processing_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap index d987d66c0..08cda75e5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap index d1369460f..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap index 03d4e5b16..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap index 136777fcf..6bf7eff90 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap index 0b4fc96b5..7c667807c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap index d938ca288..26a1524a8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/five_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: false }} 3 {uid: 3, batch_uid: 3, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap index 2d936ba68..a9fcb2536 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap index 6f0f9c782..3bc630256 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_cant_create_index_with_index_without_autobatching/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = false ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap index 6add8a2a5..86895e12c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/after_registering_the_10_tasks.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap index 197ed0679..bb769598c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/all_tasks_processed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: false }} 2 {uid: 2, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }} 3 {uid: 3, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000002, documents_count: 1, allow_index_creation: false }} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap index d1369460f..d04a2b5a0 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/processed_the_first_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_document_addition.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap index 03d4e5b16..1ec7bb770 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_document_addition.rs/test_document_addition_mixed_rights_with_index/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_document_addition.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} +0 {uid: 0, status: enqueued, details: { primary_key: None, old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggos", primary_key: None }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap index 4ece15b13..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/after_register.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap index 8feeaf990..72c638b4d 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap index 201680d7a..6b1e7fa32 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/index_creation_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "An unexpected crash occurred when processing the task: simulated panic", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap index 4ece15b13..ec0ca5d80 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/panic_in_process_batch_for_index_creation/registered_the_first_task.snap @@ -1,13 +1,12 @@ --- source: crates/index-scheduler/src/scheduler/test_failure.rs -snapshot_kind: text --- ### Autobatching Enabled = true ### Processing batch None: [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index d700dd3db..6cf99a99f 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,11 +6,11 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} ---------------------------------------------------------------------- ### Status: enqueued [] @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap index fb682053c..54b7e34e1 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_removing_the_upgrade_tasks.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves") }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} +1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +4 {uid: 4, batch_uid: 4, status: succeeded, details: { primary_key: Some("leaves"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "girafo", primary_key: Some("leaves") }} 5 {uid: 5, batch_uid: 5, status: succeeded, details: { matched_tasks: 1, deleted_tasks: Some(1), original_filter: "types=upgradeDatabase" }, kind: TaskDeletion { query: "types=upgradeDatabase", tasks: RoaringBitmap<[0]> }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index ee3cefba4..c99cd3765 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index abaffbb1b..f0fab690e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [0,1,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 9569ecfe3..05c601a41 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,8 +6,8 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: enqueued [1,] @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 1d7945023..9639fd060 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,9 +6,9 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,] @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 869d1d0b2..a14fed454 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,10 +6,10 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} -1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} -2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} -3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 18, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} +2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} +3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- ### Status: enqueued [1,2,3,] @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.18.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/test.rs b/crates/index-scheduler/src/scheduler/test.rs index e9f21dfe4..8cc1b8830 100644 --- a/crates/index-scheduler/src/scheduler/test.rs +++ b/crates/index-scheduler/src/scheduler/test.rs @@ -372,8 +372,8 @@ fn swap_indexes() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "d".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "d".to_owned()), rename: false }, ], }, None, @@ -384,7 +384,7 @@ fn swap_indexes() { index_scheduler .register( KindWithContent::IndexSwap { - swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()) }], + swaps: vec![IndexSwap { indexes: ("a".to_owned(), "c".to_owned()), rename: false }], }, None, false, @@ -428,8 +428,8 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("b".to_owned(), "a".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("b".to_owned(), "a".to_owned()), rename: false }, ], }, None, @@ -446,9 +446,9 @@ fn swap_indexes_errors() { .register( KindWithContent::IndexSwap { swaps: vec![ - IndexSwap { indexes: ("a".to_owned(), "b".to_owned()) }, - IndexSwap { indexes: ("c".to_owned(), "e".to_owned()) }, - IndexSwap { indexes: ("d".to_owned(), "f".to_owned()) }, + IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }, + IndexSwap { indexes: ("c".to_owned(), "e".to_owned()), rename: false }, + IndexSwap { indexes: ("d".to_owned(), "f".to_owned()), rename: false }, ], }, None, diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index a749b31d5..1743c3940 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -40,6 +40,8 @@ pub fn upgrade_index_scheduler( (1, 14, _) => 0, (1, 15, _) => 0, (1, 16, _) => 0, + (1, 17, _) => 0, + (1, 18, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) @@ -89,6 +91,7 @@ pub fn upgrade_index_scheduler( details: Some(Details::UpgradeDatabase { from, to }), status: Status::Enqueued, kind: KindWithContent::UpgradeDatabase { from }, + network: None, }, )?; wtxn.commit()?; diff --git a/crates/index-scheduler/src/utils.rs b/crates/index-scheduler/src/utils.rs index 3c921f099..2617aba99 100644 --- a/crates/index-scheduler/src/utils.rs +++ b/crates/index-scheduler/src/utils.rs @@ -1,6 +1,5 @@ //! Utility functions on the DBs. Mainly getter and setters. -use crate::milli::progress::EmbedderStats; use std::collections::{BTreeSet, HashSet}; use std::ops::Bound; use std::sync::Arc; @@ -15,6 +14,7 @@ use meilisearch_types::tasks::{ use roaring::RoaringBitmap; use time::OffsetDateTime; +use crate::milli::progress::EmbedderStats; use crate::{Error, Result, Task, TaskId, BEI128}; /// This structure contains all the information required to write a batch in the database without reading the tasks. @@ -264,9 +264,14 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { K::SettingsUpdate { index_uid, .. } => index_uids.push(index_uid), K::IndexDeletion { index_uid } => index_uids.push(index_uid), K::IndexCreation { index_uid, .. } => index_uids.push(index_uid), - K::IndexUpdate { index_uid, .. } => index_uids.push(index_uid), + K::IndexUpdate { index_uid, new_index_uid, .. } => { + index_uids.push(index_uid); + if let Some(new_uid) = new_index_uid { + index_uids.push(new_uid); + } + } K::IndexSwap { swaps } => { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -283,7 +288,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) { | K::SnapshotCreation => (), }; if let Some(Details::IndexSwap { swaps }) = &mut task.details { - for IndexSwap { indexes: (lhs, rhs) } in swaps.iter_mut() { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps.iter_mut() { if lhs == swap.0 || lhs == swap.1 { index_uids.push(lhs); } @@ -325,7 +330,7 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> { if let KindWithContent::IndexSwap { swaps } = &task.kind { swaps } else { return Ok(()) }; let mut all_indexes = HashSet::new(); let mut duplicate_indexes = BTreeSet::new(); - for IndexSwap { indexes: (lhs, rhs) } in swaps { + for IndexSwap { indexes: (lhs, rhs), rename: _ } in swaps { for name in [lhs, rhs] { let is_new = all_indexes.insert(name); if !is_new { @@ -372,6 +377,7 @@ impl crate::IndexScheduler { details, status, kind, + network: _, } = task; assert_eq!(uid, task.uid); if task.status != Status::Enqueued { @@ -496,9 +502,9 @@ impl crate::IndexScheduler { Details::SettingsUpdate { settings: _ } => { assert_eq!(kind.as_kind(), Kind::SettingsUpdate); } - Details::IndexInfo { primary_key: pk1 } => match &kind { + Details::IndexInfo { primary_key: pk1, .. } => match &kind { KindWithContent::IndexCreation { index_uid, primary_key: pk2 } - | KindWithContent::IndexUpdate { index_uid, primary_key: pk2 } => { + | KindWithContent::IndexUpdate { index_uid, primary_key: pk2, .. } => { self.queue .tasks .index_tasks diff --git a/crates/meilisearch-types/src/enterprise_edition/mod.rs b/crates/meilisearch-types/src/enterprise_edition/mod.rs new file mode 100644 index 000000000..47047de48 --- /dev/null +++ b/crates/meilisearch-types/src/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod network; diff --git a/crates/meilisearch-types/src/enterprise_edition/network.rs b/crates/meilisearch-types/src/enterprise_edition/network.rs new file mode 100644 index 000000000..9d5c51e25 --- /dev/null +++ b/crates/meilisearch-types/src/enterprise_edition/network.rs @@ -0,0 +1,47 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::collections::BTreeMap; + +use milli::update::new::indexer::enterprise_edition::sharding::Shards; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase")] +pub struct Network { + #[serde(default, rename = "self")] + pub local: Option, + #[serde(default)] + pub remotes: BTreeMap, + #[serde(default)] + pub sharding: bool, +} + +impl Network { + pub fn shards(&self) -> Option { + if self.sharding { + let this = self.local.as_deref().expect("Inconsistent `sharding` and `self`"); + let others = self + .remotes + .keys() + .filter(|name| name.as_str() != this) + .map(|name| name.to_owned()) + .collect(); + Some(Shards { own: vec![this.to_owned()], others }) + } else { + None + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Remote { + pub url: String, + #[serde(default)] + pub search_api_key: Option, + #[serde(default)] + pub write_api_key: Option, +} diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 1cb09eee3..93d524bda 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -212,7 +212,6 @@ ImmutableApiKeyKey , InvalidRequest , BAD_REQU ImmutableApiKeyUid , InvalidRequest , BAD_REQUEST; ImmutableApiKeyUpdatedAt , InvalidRequest , BAD_REQUEST; ImmutableIndexCreatedAt , InvalidRequest , BAD_REQUEST; -ImmutableIndexUid , InvalidRequest , BAD_REQUEST; ImmutableIndexUpdatedAt , InvalidRequest , BAD_REQUEST; IndexAlreadyExists , InvalidRequest , CONFLICT ; IndexCreationFailed , Internal , INTERNAL_SERVER_ERROR; @@ -236,9 +235,11 @@ InvalidDocumentFields , InvalidRequest , BAD_REQU InvalidDocumentRetrieveVectors , InvalidRequest , BAD_REQUEST ; MissingDocumentFilter , InvalidRequest , BAD_REQUEST ; MissingDocumentEditionFunction , InvalidRequest , BAD_REQUEST ; +InconsistentDocumentChangeHeaders , InvalidRequest , BAD_REQUEST ; InvalidDocumentFilter , InvalidRequest , BAD_REQUEST ; InvalidDocumentSort , InvalidRequest , BAD_REQUEST ; InvalidDocumentGeoField , InvalidRequest , BAD_REQUEST ; +InvalidHeaderValue , InvalidRequest , BAD_REQUEST ; InvalidVectorDimensions , InvalidRequest , BAD_REQUEST ; InvalidVectorsType , InvalidRequest , BAD_REQUEST ; InvalidDocumentId , InvalidRequest , BAD_REQUEST ; @@ -267,7 +268,9 @@ InvalidMultiSearchRemote , InvalidRequest , BAD_REQU InvalidMultiSearchWeight , InvalidRequest , BAD_REQUEST ; InvalidNetworkRemotes , InvalidRequest , BAD_REQUEST ; InvalidNetworkSelf , InvalidRequest , BAD_REQUEST ; +InvalidNetworkSharding , InvalidRequest , BAD_REQUEST ; InvalidNetworkSearchApiKey , InvalidRequest , BAD_REQUEST ; +InvalidNetworkWriteApiKey , InvalidRequest , BAD_REQUEST ; InvalidNetworkUrl , InvalidRequest , BAD_REQUEST ; InvalidSearchAttributesToSearchOn , InvalidRequest , BAD_REQUEST ; InvalidSearchAttributesToCrop , InvalidRequest , BAD_REQUEST ; @@ -335,6 +338,7 @@ InvalidState , Internal , INTERNAL InvalidStoreFile , Internal , INTERNAL_SERVER_ERROR ; InvalidSwapDuplicateIndexFound , InvalidRequest , BAD_REQUEST ; InvalidSwapIndexes , InvalidRequest , BAD_REQUEST ; +InvalidSwapRename , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterEnqueuedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterFinishedAt , InvalidRequest , BAD_REQUEST ; InvalidTaskAfterStartedAt , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index ddffb107c..cf66422b2 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use crate::error::{Code, ResponseError}; @@ -32,23 +30,6 @@ pub struct InstanceTogglableFeatures { pub contains_filter: bool, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct Remote { - pub url: String, - #[serde(default)] - pub search_api_key: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] -#[serde(rename_all = "camelCase")] -pub struct Network { - #[serde(default, rename = "self")] - pub local: Option, - #[serde(default)] - pub remotes: BTreeMap, -} - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] #[serde(rename_all = "camelCase")] pub struct ChatCompletionSettings { diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index 9857bfb29..6c013f3bb 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -3,6 +3,7 @@ pub mod batches; pub mod compression; pub mod deserr; pub mod document_formats; +pub mod enterprise_edition; pub mod error; pub mod facet_values_sort; pub mod features; diff --git a/crates/meilisearch-types/src/task_view.rs b/crates/meilisearch-types/src/task_view.rs index 7521137c0..cbc29a11b 100644 --- a/crates/meilisearch-types/src/task_view.rs +++ b/crates/meilisearch-types/src/task_view.rs @@ -11,6 +11,7 @@ use crate::error::ResponseError; use crate::settings::{Settings, Unchecked}; use crate::tasks::{ serialize_duration, Details, DetailsExportIndexSettings, IndexSwap, Kind, Status, Task, TaskId, + TaskNetwork, }; #[derive(Debug, Clone, PartialEq, Serialize, ToSchema)] @@ -51,6 +52,9 @@ pub struct TaskView { #[schema(value_type = String, example = json!("2024-08-08_14:12:09.393Z"))] #[serde(with = "time::serde::rfc3339::option", default)] pub finished_at: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } impl TaskView { @@ -68,6 +72,7 @@ impl TaskView { enqueued_at: task.enqueued_at, started_at: task.started_at, finished_at: task.finished_at, + network: task.network.clone(), } } } @@ -132,6 +137,11 @@ pub struct DetailsView { pub payload_size: Option, #[serde(skip_serializing_if = "Option::is_none")] pub indexes: Option>, + // index rename + #[serde(skip_serializing_if = "Option::is_none")] + pub old_index_uid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_index_uid: Option, } impl DetailsView { @@ -292,6 +302,18 @@ impl DetailsView { (None, Some(to)) | (Some(to), None) => Some(to), (Some(_), Some(to)) => Some(to), }, + old_index_uid: match (self.old_index_uid.clone(), other.old_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, + new_index_uid: match (self.new_index_uid.clone(), other.new_index_uid.clone()) { + (None, None) => None, + (None, Some(uid)) | (Some(uid), None) => Some(uid), + // We should never be able to batch multiple renames at the same time. + (Some(left), Some(_right)) => Some(left), + }, } } } @@ -324,9 +346,12 @@ impl From
for DetailsView { settings.hide_secrets(); DetailsView { settings: Some(settings), ..DetailsView::default() } } - Details::IndexInfo { primary_key } => { - DetailsView { primary_key: Some(primary_key), ..DetailsView::default() } - } + Details::IndexInfo { primary_key, new_index_uid, old_index_uid } => DetailsView { + primary_key: Some(primary_key), + new_index_uid: new_index_uid.clone(), + old_index_uid: old_index_uid.clone(), + ..DetailsView::default() + }, Details::DocumentDeletion { provided_ids: received_document_ids, deleted_documents, diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index 99b04f1e3..d0f668255 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -42,6 +42,9 @@ pub struct Task { pub status: Status, pub kind: KindWithContent, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, } impl Task { @@ -140,6 +143,7 @@ pub enum KindWithContent { IndexUpdate { index_uid: String, primary_key: Option, + new_index_uid: Option, }, IndexSwap { swaps: Vec, @@ -172,6 +176,8 @@ pub enum KindWithContent { #[serde(rename_all = "camelCase")] pub struct IndexSwap { pub indexes: (String, String), + #[serde(default)] + pub rename: bool, } #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] @@ -220,8 +226,14 @@ impl KindWithContent { | DocumentClear { index_uid } | SettingsUpdate { index_uid, .. } | IndexCreation { index_uid, .. } - | IndexUpdate { index_uid, .. } | IndexDeletion { index_uid } => vec![index_uid], + IndexUpdate { index_uid, new_index_uid, .. } => { + let mut indexes = vec![index_uid.as_str()]; + if let Some(new_uid) = new_index_uid { + indexes.push(new_uid.as_str()); + } + indexes + } IndexSwap { swaps } => { let mut indexes = HashSet::<&str>::default(); for swap in swaps { @@ -270,9 +282,17 @@ impl KindWithContent { KindWithContent::SettingsUpdate { new_settings, .. } => { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), + }) } KindWithContent::IndexSwap { swaps } => { Some(Details::IndexSwap { swaps: swaps.clone() }) @@ -344,9 +364,17 @@ impl KindWithContent { Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } - | KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: None, + new_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), + }) } KindWithContent::IndexSwap { .. } => { todo!() @@ -400,11 +428,17 @@ impl From<&KindWithContent> for Option
{ Some(Details::SettingsUpdate { settings: new_settings.clone() }) } KindWithContent::IndexDeletion { .. } => None, - KindWithContent::IndexCreation { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) - } - KindWithContent::IndexUpdate { primary_key, .. } => { - Some(Details::IndexInfo { primary_key: primary_key.clone() }) + KindWithContent::IndexCreation { primary_key, .. } => Some(Details::IndexInfo { + primary_key: primary_key.clone(), + new_index_uid: None, + old_index_uid: None, + }), + KindWithContent::IndexUpdate { primary_key, new_index_uid, index_uid } => { + Some(Details::IndexInfo { + primary_key: primary_key.clone(), + old_index_uid: new_index_uid.as_ref().map(|_| index_uid.clone()), + new_index_uid: new_index_uid.clone(), + }) } KindWithContent::IndexSwap { .. } => None, KindWithContent::TaskCancelation { query, tasks } => Some(Details::TaskCancelation { @@ -657,6 +691,8 @@ pub enum Details { }, IndexInfo { primary_key: Option, + new_index_uid: Option, + old_index_uid: Option, }, DocumentDeletion { provided_ids: usize, @@ -704,6 +740,36 @@ pub enum Details { }, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(untagged, rename_all = "camelCase")] +pub enum TaskNetwork { + Origin { origin: Origin }, + Remotes { remote_tasks: BTreeMap }, +} +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct Origin { + pub remote_name: String, + pub task_uid: usize, +} + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RemoteTask { + #[serde(skip_serializing_if = "Option::is_none")] + task_uid: Option, + error: Option, +} + +impl From> for RemoteTask { + fn from(res: Result) -> RemoteTask { + match res { + Ok(task_uid) => RemoteTask { task_uid: Some(task_uid), error: None }, + Err(err) => RemoteTask { task_uid: None, error: Some(err) }, + } + } +} + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)] #[schema(rename_all = "camelCase")] pub struct DetailsExportIndexSettings { diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 5cbbb6666..c110830c4 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -115,6 +115,9 @@ utoipa-scalar = { version = "0.3.0", optional = true, features = ["actix-web"] } async-openai = { git = "https://github.com/meilisearch/async-openai", branch = "better-error-handling" } secrecy = "0.10.3" actix-web-lab = { version = "0.24.1", default-features = false } +urlencoding = "2.1.3" +backoff = { version = "0.4.0", features = ["tokio"] } + [dev-dependencies] actix-rt = "2.10.0" @@ -125,7 +128,6 @@ manifest-dir-macros = "0.1.18" maplit = "1.0.2" meili-snap = { path = "../meili-snap" } temp-env = "0.3.6" -urlencoding = "2.1.3" wiremock = "0.6.3" yaup = "0.3.1" diff --git a/crates/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs index 8d4430f07..c90c7c226 100644 --- a/crates/meilisearch/src/error.rs +++ b/crates/meilisearch/src/error.rs @@ -9,6 +9,8 @@ use meilisearch_types::milli::OrderBy; use serde_json::Value; use tokio::task::JoinError; +use crate::routes::indexes::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; + #[derive(Debug, thiserror::Error)] pub enum MeilisearchHttpError { #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}", @@ -80,6 +82,16 @@ pub enum MeilisearchHttpError { MissingSearchHybrid, #[error("Invalid request: both `media` and `vector` parameters are present.")] MediaAndVector, + #[error("Inconsistent `Origin` headers: {} was provided but {} is missing.\n - Hint: Either both headers should be provided, or none of them", if *is_remote_missing { + PROXY_ORIGIN_TASK_UID_HEADER + } else { PROXY_ORIGIN_REMOTE_HEADER }, + if *is_remote_missing { + PROXY_ORIGIN_REMOTE_HEADER + } else { PROXY_ORIGIN_TASK_UID_HEADER } +)] + InconsistentOriginHeaders { is_remote_missing: bool }, + #[error("Invalid value for header {header_name}: {msg}")] + InvalidHeaderValue { header_name: &'static str, msg: String }, } impl MeilisearchHttpError { @@ -124,6 +136,10 @@ impl ErrorCode for MeilisearchHttpError { MeilisearchHttpError::InconsistentFacetOrder { .. } => { Code::InvalidMultiSearchFacetOrder } + MeilisearchHttpError::InconsistentOriginHeaders { .. } => { + Code::InconsistentDocumentChangeHeaders + } + MeilisearchHttpError::InvalidHeaderValue { .. } => Code::InvalidHeaderValue, } } } diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index ca9bb6f50..c24f81742 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -628,6 +628,7 @@ fn import_dump( &mut new_fields_ids_map, &|| false, // never stop processing a dump progress.clone(), + None, )?; let operation_stats = operation_stats.pop().unwrap(); diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 5ced4603e..d445153ad 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -45,6 +45,7 @@ use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::indexes::enterprise_edition::proxy::{proxy, Body}; use crate::routes::indexes::search::fix_sort_query_parameters; use crate::routes::{ get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, @@ -338,6 +339,7 @@ pub async fn delete_document( ) -> Result { let DocumentParam { index_uid, document_id } = path.into_inner(); let index_uid = IndexUid::try_from(index_uid)?; + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -355,10 +357,16 @@ pub async fn delete_document( }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::none(), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!("returns: {:?}", task); Ok(HttpResponse::Accepted().json(task)) } @@ -804,7 +812,6 @@ pub async fn replace_documents( let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; let task = document_addition( - extract_mime_type(&req)?, index_scheduler, index_uid, params.primary_key, @@ -814,8 +821,10 @@ pub async fn replace_documents( uid, dry_run, allow_index_creation, + &req, ) .await?; + debug!(returns = ?task, "Replace documents"); Ok(HttpResponse::Accepted().json(task)) @@ -905,7 +914,6 @@ pub async fn update_documents( let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; let task = document_addition( - extract_mime_type(&req)?, index_scheduler, index_uid, params.primary_key, @@ -915,6 +923,7 @@ pub async fn update_documents( uid, dry_run, allow_index_creation, + &req, ) .await?; debug!(returns = ?task, "Update documents"); @@ -924,7 +933,6 @@ pub async fn update_documents( #[allow(clippy::too_many_arguments)] async fn document_addition( - mime_type: Option, index_scheduler: GuardedData, Data>, index_uid: IndexUid, primary_key: Option, @@ -934,7 +942,11 @@ async fn document_addition( task_id: Option, dry_run: bool, allow_index_creation: bool, + req: &HttpRequest, ) -> Result { + let mime_type = extract_mime_type(req)?; + let network = index_scheduler.network(); + let format = match ( mime_type.as_ref().map(|m| (m.type_().as_str(), m.subtype().as_str())), csv_delimiter, @@ -966,7 +978,7 @@ async fn document_addition( }; let (uuid, mut update_file) = index_scheduler.queue.create_update_file(dry_run)?; - let documents_count = match format { + let res = match format { PayloadType::Ndjson => { let (path, file) = update_file.into_parts(); let file = match file { @@ -981,19 +993,19 @@ async fn document_addition( None => None, }; - let documents_count = tokio::task::spawn_blocking(move || { + let res = tokio::task::spawn_blocking(move || { let documents_count = file.as_ref().map_or(Ok(0), |ntf| { read_ndjson(ntf.as_file()).map_err(MeilisearchHttpError::DocumentFormat) })?; let update_file = file_store::File::from_parts(path, file); - update_file.persist()?; + let update_file = update_file.persist()?; - Ok(documents_count) + Ok((documents_count, update_file)) }) .await?; - Ok(documents_count) + Ok(res) } PayloadType::Json | PayloadType::Csv { delimiter: _ } => { let temp_file = match tempfile() { @@ -1012,16 +1024,16 @@ async fn document_addition( unreachable!("We already wrote the user content into the update file") } }; - // we NEED to persist the file here because we moved the `udpate_file` in another task. - update_file.persist()?; - Ok(documents_count) + // we NEED to persist the file here because we moved the `update_file` in another task. + let file = update_file.persist()?; + Ok((documents_count, file)) }) .await } }; - let documents_count = match documents_count { - Ok(Ok(documents_count)) => documents_count, + let (documents_count, file) = match res { + Ok(Ok((documents_count, file))) => (documents_count, file), // in this case the file has not possibly be persisted. Ok(Err(e)) => return Err(e), Err(e) => { @@ -1063,6 +1075,20 @@ async fn document_addition( } }; + if network.sharding { + if let Some(file) = file { + proxy( + &index_scheduler, + &index_uid, + req, + network, + Body::with_ndjson_payload(file), + &task, + ) + .await?; + } + } + Ok(task.into()) } @@ -1141,6 +1167,7 @@ pub async fn delete_documents_batch( ) -> Result { debug!(parameters = ?body, "Delete documents by batch"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -1161,16 +1188,22 @@ pub async fn delete_documents_batch( KindWithContent::DocumentDeletion { index_uid: index_uid.to_string(), documents_ids: ids }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(body), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete documents by batch"); Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, ToSchema)] +#[derive(Debug, Deserr, ToSchema, Serialize)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[schema(rename_all = "camelCase")] pub struct DocumentDeletionByFilter { @@ -1219,7 +1252,8 @@ pub async fn delete_documents_by_filter( debug!(parameters = ?body, "Delete documents by filter"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let index_uid = index_uid.into_inner(); - let filter = body.into_inner().filter; + let filter = body.into_inner(); + let network = index_scheduler.network(); analytics.publish( DocumentsDeletionAggregator { @@ -1232,23 +1266,36 @@ pub async fn delete_documents_by_filter( ); // we ensure the filter is well formed before enqueuing it - crate::search::parse_filter(&filter, Code::InvalidDocumentFilter, index_scheduler.features())? - .ok_or(MeilisearchHttpError::EmptyFilter)?; + crate::search::parse_filter( + &filter.filter, + Code::InvalidDocumentFilter, + index_scheduler.features(), + )? + .ok_or(MeilisearchHttpError::EmptyFilter)?; - let task = KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr: filter }; + let task = KindWithContent::DocumentDeletionByFilter { + index_uid: index_uid.clone(), + filter_expr: filter.filter.clone(), + }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(filter), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete documents by filter"); Ok(HttpResponse::Accepted().json(task)) } -#[derive(Debug, Deserr, ToSchema)] +#[derive(Debug, Deserr, ToSchema, Serialize)] #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] pub struct DocumentEditionByFunction { /// A string containing a RHAI function. @@ -1336,6 +1383,8 @@ pub async fn edit_documents_by_function( .features() .check_edit_documents_by_function("Using the documents edit route")?; + let network = index_scheduler.network(); + let index_uid = IndexUid::try_from(index_uid.into_inner())?; let index_uid = index_uid.into_inner(); let params = params.into_inner(); @@ -1349,13 +1398,12 @@ pub async fn edit_documents_by_function( &req, ); - let DocumentEditionByFunction { filter, context, function } = params; let engine = milli::rhai::Engine::new(); - if let Err(e) = engine.compile(&function) { + if let Err(e) = engine.compile(¶ms.function) { return Err(ResponseError::from_msg(e.to_string(), Code::BadRequest)); } - if let Some(ref filter) = filter { + if let Some(ref filter) = params.filter { // we ensure the filter is well formed before enqueuing it crate::search::parse_filter( filter, @@ -1365,9 +1413,9 @@ pub async fn edit_documents_by_function( .ok_or(MeilisearchHttpError::EmptyFilter)?; } let task = KindWithContent::DocumentEdition { - index_uid, - filter_expr: filter, - context: match context { + index_uid: index_uid.clone(), + filter_expr: params.filter.clone(), + context: match params.context.clone() { Some(Value::Object(m)) => Some(m), None => None, _ => { @@ -1377,15 +1425,21 @@ pub async fn edit_documents_by_function( )) } }, - function, + function: params.function.clone(), }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + let task = { + let index_scheduler = index_scheduler.clone(); + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::Inline(params), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Edit documents by function"); Ok(HttpResponse::Accepted().json(task)) @@ -1428,6 +1482,8 @@ pub async fn clear_all_documents( analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; + let network = index_scheduler.network(); + analytics.publish( DocumentsDeletionAggregator { clear_all: true, @@ -1441,10 +1497,18 @@ pub async fn clear_all_documents( let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() }; let uid = get_task_id(&req, &opt)?; let dry_run = is_dry_run(&req, &opt)?; - let task: SummarizedTaskView = - tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)) - .await?? - .into(); + + let task = { + let index_scheduler = index_scheduler.clone(); + + tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run)).await?? + }; + + if network.sharding && !dry_run { + proxy(&index_scheduler, &index_uid, &req, network, Body::none(), &task).await?; + } + + let task: SummarizedTaskView = task.into(); debug!(returns = ?task, "Delete all documents"); Ok(HttpResponse::Accepted().json(task)) diff --git a/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs b/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs new file mode 100644 index 000000000..a3ca7350c --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod proxy; diff --git a/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs new file mode 100644 index 000000000..7cb3eb8cc --- /dev/null +++ b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs @@ -0,0 +1,426 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::collections::BTreeMap; +use std::fs::File; + +use actix_web::http::header::CONTENT_TYPE; +use actix_web::HttpRequest; +use bytes::Bytes; +use index_scheduler::IndexScheduler; +use meilisearch_types::error::ResponseError; +use meilisearch_types::tasks::{Origin, RemoteTask, TaskNetwork}; +use reqwest::StatusCode; +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::error::MeilisearchHttpError; +use crate::routes::indexes::enterprise_edition::proxy::error::{ + ProxyDocumentChangeError, ReqwestErrorWithoutUrl, +}; +use crate::routes::SummarizedTaskView; + +pub enum Body { + NdJsonPayload(File), + Inline(T), + None, +} + +impl Body<()> { + pub fn with_ndjson_payload(file: File) -> Self { + Self::NdJsonPayload(file) + } + + pub fn none() -> Self { + Self::None + } +} + +/// If necessary, proxies the passed request to the network and update the task description. +/// +/// This function reads the custom headers from the request to determine if must proxy the request or if the request +/// has already been proxied. +/// +/// - when it must proxy the request, the endpoint, method and query params are retrieved from the passed `req`, then the `body` is +/// sent to all remotes of the `network` (except `self`). The response from the remotes are collected to update the passed `task` +/// with the task ids from the task queues of the remotes. +/// - when the request has already been proxied, the custom headers contains information about the remote that created the initial task. +/// This information is copied to the passed task. +pub async fn proxy( + index_scheduler: &IndexScheduler, + index_uid: &str, + req: &HttpRequest, + network: meilisearch_types::enterprise_edition::network::Network, + body: Body, + task: &meilisearch_types::tasks::Task, +) -> Result<(), MeilisearchHttpError> { + match origin_from_req(req)? { + Some(origin) => { + index_scheduler.set_task_network(task.uid, TaskNetwork::Origin { origin })? + } + None => { + let this = network + .local + .as_deref() + .expect("inconsistent `network.sharding` and `network.self`") + .to_owned(); + + let content_type = match &body { + // for file bodies, force x-ndjson + Body::NdJsonPayload(_) => Some(b"application/x-ndjson".as_slice()), + // otherwise get content type from request + _ => req.headers().get(CONTENT_TYPE).map(|h| h.as_bytes()), + }; + + let body = match body { + Body::NdJsonPayload(file) => Some(Bytes::from_owner(unsafe { + memmap2::Mmap::map(&file).map_err(|err| { + MeilisearchHttpError::from_milli(err.into(), Some(index_uid.to_owned())) + })? + })), + + Body::Inline(payload) => { + Some(Bytes::copy_from_slice(&serde_json::to_vec(&payload).unwrap())) + } + + Body::None => None, + }; + + let mut in_flight_remote_queries = BTreeMap::new(); + let client = reqwest::ClientBuilder::new() + .connect_timeout(std::time::Duration::from_secs(3)) + .build() + .unwrap(); + + let method = from_old_http_method(req.method()); + + // send payload to all remotes + for (node_name, node) in + network.remotes.into_iter().filter(|(name, _)| name.as_str() != this) + { + let body = body.clone(); + let client = client.clone(); + let api_key = node.write_api_key; + let this = this.clone(); + let method = method.clone(); + let path_and_query = + req.uri().path_and_query().map(|paq| paq.as_str()).unwrap_or("/"); + + in_flight_remote_queries.insert( + node_name, + tokio::spawn({ + let url = format!("{}{}", node.url, path_and_query); + + let url_encoded_this = urlencoding::encode(&this).into_owned(); + let url_encoded_task_uid = task.uid.to_string(); // it's url encoded i promize + + let content_type = content_type.map(|b| b.to_owned()); + + let backoff = backoff::ExponentialBackoffBuilder::new() + .with_max_elapsed_time(Some(std::time::Duration::from_secs(25))) + .build(); + + backoff::future::retry(backoff, move || { + let url = url.clone(); + let client = client.clone(); + let url_encoded_this = url_encoded_this.clone(); + let url_encoded_task_uid = url_encoded_task_uid.clone(); + let content_type = content_type.clone(); + + let body = body.clone(); + let api_key = api_key.clone(); + let method = method.clone(); + + async move { + try_proxy( + method, + &url, + content_type.as_deref(), + api_key.as_deref(), + &client, + &url_encoded_this, + &url_encoded_task_uid, + body, + ) + .await + } + }) + }), + ); + } + + // wait for all in-flight queries to finish and collect their results + let mut remote_tasks: BTreeMap = BTreeMap::new(); + for (node_name, handle) in in_flight_remote_queries { + match handle.await { + Ok(Ok(res)) => { + let task_uid = res.task_uid; + + remote_tasks.insert(node_name, Ok(task_uid).into()); + } + Ok(Err(error)) => { + remote_tasks.insert(node_name, Err(error.as_response_error()).into()); + } + Err(panic) => match panic.try_into_panic() { + Ok(panic) => { + let msg = match panic.downcast_ref::<&'static str>() { + Some(s) => *s, + None => match panic.downcast_ref::() { + Some(s) => &s[..], + None => "Box", + }, + }; + remote_tasks.insert( + node_name, + Err(ResponseError::from_msg( + msg.to_string(), + meilisearch_types::error::Code::Internal, + )) + .into(), + ); + } + Err(_) => { + tracing::error!("proxy task was unexpectedly cancelled") + } + }, + } + } + + // edit details to contain the return values from the remotes + index_scheduler.set_task_network(task.uid, TaskNetwork::Remotes { remote_tasks })?; + } + } + + Ok(()) +} + +fn from_old_http_method(method: &actix_http::Method) -> reqwest::Method { + match method { + &actix_http::Method::CONNECT => reqwest::Method::CONNECT, + &actix_http::Method::DELETE => reqwest::Method::DELETE, + &actix_http::Method::GET => reqwest::Method::GET, + &actix_http::Method::HEAD => reqwest::Method::HEAD, + &actix_http::Method::OPTIONS => reqwest::Method::OPTIONS, + &actix_http::Method::PATCH => reqwest::Method::PATCH, + &actix_http::Method::POST => reqwest::Method::POST, + &actix_http::Method::PUT => reqwest::Method::PUT, + &actix_http::Method::TRACE => reqwest::Method::TRACE, + method => reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap(), + } +} + +#[allow(clippy::too_many_arguments)] +async fn try_proxy( + method: reqwest::Method, + url: &str, + content_type: Option<&[u8]>, + api_key: Option<&str>, + client: &reqwest::Client, + url_encoded_this: &str, + url_encoded_task_uid: &str, + body: Option, +) -> Result> { + let request = client.request(method, url).timeout(std::time::Duration::from_secs(30)); + let request = if let Some(body) = body { request.body(body) } else { request }; + let request = if let Some(api_key) = api_key { request.bearer_auth(api_key) } else { request }; + let request = request.header(PROXY_ORIGIN_TASK_UID_HEADER, url_encoded_task_uid); + let request = request.header(PROXY_ORIGIN_REMOTE_HEADER, url_encoded_this); + let request = if let Some(content_type) = content_type { + request.header(CONTENT_TYPE.as_str(), content_type) + } else { + request + }; + + let response = request.send().await; + let response = match response { + Ok(response) => response, + Err(error) if error.is_timeout() => { + return Err(backoff::Error::transient(ProxyDocumentChangeError::Timeout)) + } + Err(error) => { + return Err(backoff::Error::transient(ProxyDocumentChangeError::CouldNotSendRequest( + ReqwestErrorWithoutUrl::new(error), + ))) + } + }; + + match response.status() { + status_code if status_code.is_success() => (), + StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => { + return Err(backoff::Error::Permanent(ProxyDocumentChangeError::AuthenticationError)) + } + status_code if status_code.is_client_error() => { + let response = parse_error(response).await; + return Err(backoff::Error::Permanent(ProxyDocumentChangeError::BadRequest { + status_code, + response, + })); + } + status_code if status_code.is_server_error() => { + let response = parse_error(response).await; + return Err(backoff::Error::transient(ProxyDocumentChangeError::RemoteError { + status_code, + response, + })); + } + status_code => { + tracing::warn!( + status_code = status_code.as_u16(), + "remote replied with unexpected status code" + ); + } + } + + let response = match parse_response(response).await { + Ok(response) => response, + Err(response) => { + return Err(backoff::Error::transient( + ProxyDocumentChangeError::CouldNotParseResponse { response }, + )) + } + }; + + Ok(response) +} + +async fn parse_error(response: reqwest::Response) -> Result { + let bytes = match response.bytes().await { + Ok(bytes) => bytes, + Err(error) => return Err(ReqwestErrorWithoutUrl::new(error)), + }; + + Ok(parse_bytes_as_error(&bytes)) +} + +fn parse_bytes_as_error(bytes: &[u8]) -> String { + match serde_json::from_slice::(bytes) { + Ok(value) => value.to_string(), + Err(_) => String::from_utf8_lossy(bytes).into_owned(), + } +} + +async fn parse_response( + response: reqwest::Response, +) -> Result> { + let bytes = match response.bytes().await { + Ok(bytes) => bytes, + Err(error) => return Err(Err(ReqwestErrorWithoutUrl::new(error))), + }; + + match serde_json::from_slice::(&bytes) { + Ok(value) => Ok(value), + Err(_) => Err(Ok(parse_bytes_as_error(&bytes))), + } +} + +mod error { + use meilisearch_types::error::ResponseError; + use reqwest::StatusCode; + + #[derive(Debug, thiserror::Error)] + pub enum ProxyDocumentChangeError { + #[error("{0}")] + CouldNotSendRequest(ReqwestErrorWithoutUrl), + #[error("could not authenticate against the remote host\n - hint: check that the remote instance was registered with a valid API key having the `documents.add` action")] + AuthenticationError, + #[error( + "could not parse response from the remote host as a document addition response{}\n - hint: check that the remote instance is a Meilisearch instance running the same version", + response_from_remote(response) + )] + CouldNotParseResponse { response: Result }, + #[error("remote host responded with code {}{}\n - hint: check that the remote instance has the correct index configuration for that request\n - hint: check that the `network` experimental feature is enabled on the remote instance", status_code.as_u16(), response_from_remote(response))] + BadRequest { status_code: StatusCode, response: Result }, + #[error("remote host did not answer before the deadline")] + Timeout, + #[error("remote host responded with code {}{}", status_code.as_u16(), response_from_remote(response))] + RemoteError { status_code: StatusCode, response: Result }, + } + + impl ProxyDocumentChangeError { + pub fn as_response_error(&self) -> ResponseError { + use meilisearch_types::error::Code; + let message = self.to_string(); + let code = match self { + ProxyDocumentChangeError::CouldNotSendRequest(_) => Code::RemoteCouldNotSendRequest, + ProxyDocumentChangeError::AuthenticationError => Code::RemoteInvalidApiKey, + ProxyDocumentChangeError::BadRequest { .. } => Code::RemoteBadRequest, + ProxyDocumentChangeError::Timeout => Code::RemoteTimeout, + ProxyDocumentChangeError::RemoteError { .. } => Code::RemoteRemoteError, + ProxyDocumentChangeError::CouldNotParseResponse { .. } => Code::RemoteBadResponse, + }; + ResponseError::from_msg(message, code) + } + } + + #[derive(Debug, thiserror::Error)] + #[error(transparent)] + pub struct ReqwestErrorWithoutUrl(reqwest::Error); + impl ReqwestErrorWithoutUrl { + pub fn new(inner: reqwest::Error) -> Self { + Self(inner.without_url()) + } + } + + fn response_from_remote(response: &Result) -> String { + match response { + Ok(response) => { + format!(":\n - response from remote: {}", response) + } + Err(error) => { + format!(":\n - additionally, could not retrieve response from remote: {error}") + } + } + } +} + +pub const PROXY_ORIGIN_REMOTE_HEADER: &str = "Meili-Proxy-Origin-Remote"; +pub const PROXY_ORIGIN_TASK_UID_HEADER: &str = "Meili-Proxy-Origin-TaskUid"; + +pub fn origin_from_req(req: &HttpRequest) -> Result, MeilisearchHttpError> { + let (remote_name, task_uid) = match ( + req.headers().get(PROXY_ORIGIN_REMOTE_HEADER), + req.headers().get(PROXY_ORIGIN_TASK_UID_HEADER), + ) { + (None, None) => return Ok(None), + (None, Some(_)) => { + return Err(MeilisearchHttpError::InconsistentOriginHeaders { is_remote_missing: true }) + } + (Some(_), None) => { + return Err(MeilisearchHttpError::InconsistentOriginHeaders { + is_remote_missing: false, + }) + } + (Some(remote_name), Some(task_uid)) => ( + urlencoding::decode(remote_name.to_str().map_err(|err| { + MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_REMOTE_HEADER, + msg: format!("while parsing remote name as UTF-8: {err}"), + } + })?) + .map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_REMOTE_HEADER, + msg: format!("while URL-decoding remote name: {err}"), + })?, + urlencoding::decode(task_uid.to_str().map_err(|err| { + MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while parsing task UID as UTF-8: {err}"), + } + })?) + .map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while URL-decoding task UID: {err}"), + })?, + ), + }; + + let task_uid: usize = + task_uid.parse().map_err(|err| MeilisearchHttpError::InvalidHeaderValue { + header_name: PROXY_ORIGIN_TASK_UID_HEADER, + msg: format!("while parsing the task UID as an integer: {err}"), + })?; + + Ok(Some(Origin { remote_name: remote_name.into_owned(), task_uid })) +} diff --git a/crates/meilisearch/src/routes/indexes/mod.rs b/crates/meilisearch/src/routes/indexes/mod.rs index 04b3e12c4..8e994bb43 100644 --- a/crates/meilisearch/src/routes/indexes/mod.rs +++ b/crates/meilisearch/src/routes/indexes/mod.rs @@ -29,6 +29,7 @@ use crate::routes::is_dry_run; use crate::Opt; pub mod documents; +mod enterprise_edition; pub mod facet_search; pub mod search; mod search_analytics; @@ -39,6 +40,8 @@ mod settings_analytics; pub mod similar; mod similar_analytics; +pub use enterprise_edition::proxy::{PROXY_ORIGIN_REMOTE_HEADER, PROXY_ORIGIN_TASK_UID_HEADER}; + #[derive(OpenApi)] #[openapi( nest( @@ -288,7 +291,6 @@ fn deny_immutable_fields_index( location: ValuePointerRef, ) -> DeserrJsonError { match field { - "uid" => immutable_field_error(field, accepted, Code::ImmutableIndexUid), "createdAt" => immutable_field_error(field, accepted, Code::ImmutableIndexCreatedAt), "updatedAt" => immutable_field_error(field, accepted, Code::ImmutableIndexUpdatedAt), _ => deserr::take_cf_content(DeserrJsonError::::error::( @@ -375,6 +377,9 @@ pub struct UpdateIndexRequest { /// The new primary key of the index #[deserr(default, error = DeserrJsonError)] primary_key: Option, + /// The new uid of the index (for renaming) + #[deserr(default, error = DeserrJsonError)] + uid: Option, } /// Update index @@ -419,6 +424,12 @@ pub async fn update_index( debug!(parameters = ?body, "Update index"); let index_uid = IndexUid::try_from(index_uid.into_inner())?; let body = body.into_inner(); + + // Validate new uid if provided + if let Some(ref new_uid) = body.uid { + let _ = IndexUid::try_from(new_uid.clone())?; + } + analytics.publish( IndexUpdatedAggregate { primary_key: body.primary_key.iter().cloned().collect() }, &req, @@ -427,6 +438,7 @@ pub async fn update_index( let task = KindWithContent::IndexUpdate { index_uid: index_uid.into_inner(), primary_key: body.primary_key, + new_index_uid: body.uid, }; let uid = get_task_id(&req, &opt)?; diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 745ac5824..fd6e777de 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -184,7 +184,7 @@ pub fn is_dry_run(req: &HttpRequest, opt: &Opt) -> Result { .is_some_and(|s| s.to_lowercase() == "true")) } -#[derive(Debug, Serialize, ToSchema)] +#[derive(Debug, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct SummarizedTaskView { /// The task unique identifier. @@ -198,7 +198,10 @@ pub struct SummarizedTaskView { #[serde(rename = "type")] kind: Kind, /// The date on which the task was enqueued. - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde( + serialize_with = "time::serde::rfc3339::serialize", + deserialize_with = "time::serde::rfc3339::deserialize" + )] enqueued_at: OffsetDateTime, } diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 4afa32c09..b7fa2980c 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -7,11 +7,12 @@ use deserr::Deserr; use index_scheduler::IndexScheduler; use itertools::{EitherOrBoth, Itertools}; use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::enterprise_edition::network::{Network as DbNetwork, Remote as DbRemote}; use meilisearch_types::error::deserr_codes::{ - InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkUrl, + InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkSharding, + InvalidNetworkUrl, InvalidNetworkWriteApiKey, }; use meilisearch_types::error::ResponseError; -use meilisearch_types::features::{Network as DbNetwork, Remote as DbRemote}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; use serde::Serialize; @@ -57,9 +58,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { { "self": "ms-0", "remotes": { - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, } })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( @@ -88,9 +89,9 @@ async fn get_network( #[schema(rename_all = "camelCase")] pub struct Remote { #[schema(value_type = Option, example = json!({ - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, }))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -99,6 +100,10 @@ pub struct Remote { #[deserr(default, error = DeserrJsonError)] #[serde(default)] pub search_api_key: Setting, + #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub write_api_key: Setting, } #[derive(Debug, Deserr, ToSchema, Serialize)] @@ -114,6 +119,10 @@ pub struct Network { #[serde(default, rename = "self")] #[deserr(default, rename = "self", error = DeserrJsonError)] pub local: Setting, + #[schema(value_type = Option, example = json!(true))] + #[serde(default)] + #[deserr(default, error = DeserrJsonError)] + pub sharding: Setting, } impl Remote { @@ -136,6 +145,7 @@ impl Remote { Ok(url) })?, search_api_key: self.search_api_key.set(), + write_api_key: self.write_api_key.set(), }) } } @@ -174,9 +184,9 @@ impl Aggregate for PatchNetworkAnalytics { { "self": "ms-0", "remotes": { - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()) }, + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, } })), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( @@ -207,6 +217,19 @@ async fn patch_network( Setting::NotSet => old_network.local, }; + let merged_sharding = match new_network.sharding { + Setting::Set(new_sharding) => new_sharding, + Setting::Reset => false, + Setting::NotSet => old_network.sharding, + }; + + if merged_sharding && merged_self.is_none() { + return Err(ResponseError::from_msg( + "`.sharding`: enabling the sharding requires `.self` to be set\n - Hint: Disable `sharding` or set `self` to a value.".into(), + meilisearch_types::error::Code::InvalidNetworkSharding, + )); + } + let merged_remotes = match new_network.remotes { Setting::Set(new_remotes) => { let mut merged_remotes = BTreeMap::new(); @@ -217,9 +240,17 @@ async fn patch_network( { match either_or_both { EitherOrBoth::Both((key, old), (_, Some(new))) => { - let DbRemote { url: old_url, search_api_key: old_search_api_key } = old; + let DbRemote { + url: old_url, + search_api_key: old_search_api_key, + write_api_key: old_write_api_key, + } = old; - let Remote { url: new_url, search_api_key: new_search_api_key } = new; + let Remote { + url: new_url, + search_api_key: new_search_api_key, + write_api_key: new_write_api_key, + } = new; let merged = DbRemote { url: match new_url { @@ -247,6 +278,11 @@ async fn patch_network( Setting::Reset => None, Setting::NotSet => old_search_api_key, }, + write_api_key: match new_write_api_key { + Setting::Set(new_write_api_key) => Some(new_write_api_key), + Setting::Reset => None, + Setting::NotSet => old_write_api_key, + }, }; merged_remotes.insert(key, merged); } @@ -274,7 +310,8 @@ async fn patch_network( &req, ); - let merged_network = DbNetwork { local: merged_self, remotes: merged_remotes }; + let merged_network = + DbNetwork { local: merged_self, remotes: merged_remotes, sharding: merged_sharding }; index_scheduler.put_network(merged_network.clone())?; debug!(returns = ?merged_network, "Patch network"); Ok(HttpResponse::Ok().json(merged_network)) diff --git a/crates/meilisearch/src/routes/swap_indexes.rs b/crates/meilisearch/src/routes/swap_indexes.rs index 4a35d1a6d..533d2caf1 100644 --- a/crates/meilisearch/src/routes/swap_indexes.rs +++ b/crates/meilisearch/src/routes/swap_indexes.rs @@ -4,7 +4,7 @@ use deserr::actix_web::AwebJson; use deserr::Deserr; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::InvalidSwapIndexes; +use meilisearch_types::error::deserr_codes::{InvalidSwapIndexes, InvalidSwapRename}; use meilisearch_types::error::ResponseError; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::tasks::{IndexSwap, KindWithContent}; @@ -33,11 +33,15 @@ pub struct SwapIndexesPayload { /// Array of the two indexUids to be swapped #[deserr(error = DeserrJsonError, missing_field_error = DeserrJsonError::missing_swap_indexes)] indexes: Vec, + /// If set to true, instead of swapping the left and right indexes it'll change the name of the first index to the second + #[deserr(default, error = DeserrJsonError)] + rename: bool, } #[derive(Serialize)] struct IndexSwappedAnalytics { swap_operation_number: usize, + rename_used: bool, } impl Aggregate for IndexSwappedAnalytics { @@ -48,6 +52,7 @@ impl Aggregate for IndexSwappedAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(Self { swap_operation_number: self.swap_operation_number.max(new.swap_operation_number), + rename_used: self.rename_used | new.rename_used, }) } @@ -95,11 +100,17 @@ pub async fn swap_indexes( analytics: web::Data, ) -> Result { let params = params.into_inner(); - analytics.publish(IndexSwappedAnalytics { swap_operation_number: params.len() }, &req); + analytics.publish( + IndexSwappedAnalytics { + swap_operation_number: params.len(), + rename_used: params.iter().any(|obj| obj.rename), + }, + &req, + ); let filters = index_scheduler.filters(); let mut swaps = vec![]; - for SwapIndexesPayload { indexes } in params.into_iter() { + for SwapIndexesPayload { indexes, rename } in params.into_iter() { // TODO: switch to deserr let (lhs, rhs) = match indexes.as_slice() { [lhs, rhs] => (lhs, rhs), @@ -110,7 +121,7 @@ pub async fn swap_indexes( if !filters.is_index_authorized(lhs) || !filters.is_index_authorized(rhs) { return Err(AuthenticationError::InvalidToken.into()); } - swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()) }); + swaps.push(IndexSwap { indexes: (lhs.to_string(), rhs.to_string()), rename }); } let task = KindWithContent::IndexSwap { swaps }; diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 3c80c22e3..4988dc07d 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -9,8 +9,8 @@ use std::vec::{IntoIter, Vec}; use actix_http::StatusCode; use index_scheduler::{IndexScheduler, RoFeatures}; use itertools::Itertools; +use meilisearch_types::enterprise_edition::network::{Network, Remote}; use meilisearch_types::error::ResponseError; -use meilisearch_types::features::{Network, Remote}; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; use meilisearch_types::milli::vector::Embedding; diff --git a/crates/meilisearch/src/search/federated/proxy.rs b/crates/meilisearch/src/search/federated/proxy.rs index bf954693c..ac8cd2491 100644 --- a/crates/meilisearch/src/search/federated/proxy.rs +++ b/crates/meilisearch/src/search/federated/proxy.rs @@ -1,6 +1,6 @@ pub use error::ProxySearchError; use error::ReqwestErrorWithoutUrl; -use meilisearch_types::features::Remote; +use meilisearch_types::enterprise_edition::network::Remote; use rand::Rng as _; use reqwest::{Client, Response, StatusCode}; use serde::de::DeserializeOwned; diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 9d6bee7c1..4477d93e8 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1142,7 +1142,7 @@ async fn test_summarized_index_swap() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, - @r###" + @r#" { "uid": "[uid]", "progress": null, @@ -1152,7 +1152,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -1172,7 +1173,7 @@ async fn test_summarized_index_swap() { "finishedAt": "[date]", "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." } - "###); + "#); let doggos_index = server.unique_index(); doggos_index.create(None).await; diff --git a/crates/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs index 012c9bebe..f8ff5ced9 100644 --- a/crates/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -319,6 +319,24 @@ impl Index<'_, Shared> { } (task, code) } + + pub async fn update_raw_index_fail( + &self, + body: Value, + waiter: &Server, + ) -> (Value, StatusCode) { + let (mut task, code) = self._update_raw(body).await; + if code.is_success() { + task = waiter.wait_task(task.uid()).await; + if task.is_success() { + panic!( + "`update_raw_index_fail` succeeded: {}", + serde_json::to_string_pretty(&task).unwrap() + ); + } + } + (task, code) + } } #[allow(dead_code)] @@ -370,6 +388,11 @@ impl Index<'_, State> { self.service.patch_encoded(url, body, self.encoder).await } + pub(super) async fn _update_raw(&self, body: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); + self.service.patch_encoded(url, body, self.encoder).await + } + pub(super) async fn _delete(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}", urlencode(self.uid.as_ref())); self.service.delete(url).await diff --git a/crates/meilisearch/tests/index/errors.rs b/crates/meilisearch/tests/index/errors.rs index 3bab83955..2f10a5b67 100644 --- a/crates/meilisearch/tests/index/errors.rs +++ b/crates/meilisearch/tests/index/errors.rs @@ -160,36 +160,20 @@ async fn update_index_bad_primary_key() { "###); } -#[actix_rt::test] -async fn update_index_immutable_uid() { - let server = Server::new_shared(); - let index = server.unique_index(); - let (response, code) = index.update_raw(json!({ "uid": "doggo" })).await; - snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" - { - "message": "Immutable field `uid`: expected one of `primaryKey`", - "code": "immutable_index_uid", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#immutable_index_uid" - } - "###); -} - #[actix_rt::test] async fn update_index_immutable_created_at() { let server = Server::new_shared(); let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "createdAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `createdAt`: expected one of `primaryKey`", + "message": "Immutable field `createdAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_created_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_created_at" } - "###); + "#); } #[actix_rt::test] @@ -198,14 +182,14 @@ async fn update_index_immutable_updated_at() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "updatedAt": "doggo" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Immutable field `updatedAt`: expected one of `primaryKey`", + "message": "Immutable field `updatedAt`: expected one of `primaryKey`, `uid`", "code": "immutable_index_updated_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#immutable_index_updated_at" } - "###); + "#); } #[actix_rt::test] @@ -214,14 +198,14 @@ async fn update_index_unknown_field() { let index = server.unique_index(); let (response, code) = index.update_raw(json!({ "doggo": "bork" })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Unknown field `doggo`: expected one of `primaryKey`", + "message": "Unknown field `doggo`: expected one of `primaryKey`, `uid`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/index/mod.rs b/crates/meilisearch/tests/index/mod.rs index 5df5e7e97..f3cb7fde9 100644 --- a/crates/meilisearch/tests/index/mod.rs +++ b/crates/meilisearch/tests/index/mod.rs @@ -2,5 +2,6 @@ mod create_index; mod delete_index; mod errors; mod get_index; +mod rename_index; mod stats; mod update_index; diff --git a/crates/meilisearch/tests/index/rename_index.rs b/crates/meilisearch/tests/index/rename_index.rs new file mode 100644 index 000000000..23109f272 --- /dev/null +++ b/crates/meilisearch/tests/index/rename_index.rs @@ -0,0 +1,392 @@ +use crate::common::{shared_does_not_exists_index, Server}; +use crate::json; + +#[actix_rt::test] +async fn rename_index_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename via PATCH update endpoint + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); +} + +#[actix_rt::test] +async fn rename_to_existing_index_via_patch() { + let server = Server::new_shared(); + let index1 = server.unique_index(); + let index2 = server.unique_index(); + + // Create both indexes + let (task, code) = index1.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let (task, code) = index2.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename index1 to index2's uid via PATCH (should fail) + let body = json!({ "uid": index2.uid }); + let (task, code) = index1.service.patch(format!("/indexes/{}", index1.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` already exists.", index2.uid), + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_non_existent_index_via_patch() { + let server = Server::new_shared(); + let index = shared_does_not_exists_index().await; + + // Try to rename non-existent index via PATCH + let body = json!({ "uid": "new_name" }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.failed(); + + let expected_response = json!({ + "message": format!("Index `{}` not found.", index.uid), + "code": "index_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_not_found" + }); + + assert_eq!(response["error"], expected_response); +} + +#[actix_rt::test] +async fn rename_with_invalid_uid_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index first + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Try to rename with invalid uid via PATCH + let body = json!({ "uid": "Invalid UID!" }); + let (_, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 400); +} + +#[actix_rt::test] +async fn rename_index_with_documents_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1" }, + { "id": 2, "title": "Movie 2" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index via PATCH + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify documents are accessible in renamed index + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["results"].as_array().unwrap().len(), 2); +} + +#[actix_rt::test] +async fn rename_index_and_update_primary_key_via_patch() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index without primary key + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename index and set primary key at the same time + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ + "uid": &new_uid, + "primaryKey": "id" + }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + let response = server.wait_task(task.uid()).await.succeeded(); + + // Verify the rename succeeded and primary key was set + assert_eq!(response["status"], "succeeded"); + assert_eq!(response["type"], "indexUpdate"); + assert_eq!(response["details"]["newIndexUid"], new_uid); + assert_eq!(response["details"]["primaryKey"], "id"); + + // Check that old index doesn't exist + let (_, code) = index.get().await; + assert_eq!(code, 404); + + // Check that new index exists with correct primary key + let (response, code) = server.service.get(format!("/indexes/{}", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(response["uid"], new_uid); + assert_eq!(response["primaryKey"], "id"); +} + +#[actix_rt::test] +async fn rename_index_and_verify_stats() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "Movie 1", "genre": "Action" }, + { "id": 2, "title": "Movie 2", "genre": "Drama" }, + { "id": 3, "title": "Movie 3", "genre": "Comedy" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats before rename + let (stats_before, code) = index.stats().await; + assert_eq!(code, 200); + assert_eq!(stats_before["numberOfDocuments"], 3); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Get stats after rename using the new uid + let (stats_after, code) = server.service.get(format!("/indexes/{}/stats", new_uid)).await; + assert_eq!(code, 200); + assert_eq!(stats_after["numberOfDocuments"], 3); + assert_eq!(stats_after["numberOfDocuments"], stats_before["numberOfDocuments"]); +} + +#[actix_rt::test] +async fn rename_index_preserves_settings() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Configure settings + let settings = json!({ + "searchableAttributes": ["title", "description"], + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": ["the", "a", "an"], + "synonyms": { + "movie": ["film", "picture"], + "great": ["awesome", "excellent"] + } + }); + + let (task, code) = index.update_settings(settings.clone()).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify settings are preserved + let (settings_after, code) = server.service.get(format!("/indexes/{}/settings", new_uid)).await; + assert_eq!(code, 200); + + assert_eq!(settings_after["searchableAttributes"], json!(["title", "description"])); + assert_eq!(settings_after["filterableAttributes"], json!(["genre", "year"])); + assert_eq!(settings_after["sortableAttributes"], json!(["year"])); + + // Check stopWords contains the same items (order may vary) + let stop_words = settings_after["stopWords"].as_array().unwrap(); + assert_eq!(stop_words.len(), 3); + assert!(stop_words.contains(&json!("the"))); + assert!(stop_words.contains(&json!("a"))); + assert!(stop_words.contains(&json!("an"))); + + assert_eq!(settings_after["synonyms"]["movie"], json!(["film", "picture"])); + assert_eq!(settings_after["synonyms"]["great"], json!(["awesome", "excellent"])); +} + +#[actix_rt::test] +async fn rename_index_preserves_search_functionality() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index and add documents + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + let documents = json!([ + { "id": 1, "title": "The Matrix", "genre": "Sci-Fi", "year": 1999 }, + { "id": 2, "title": "Inception", "genre": "Sci-Fi", "year": 2010 }, + { "id": 3, "title": "The Dark Knight", "genre": "Action", "year": 2008 } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Make settings filterable + let settings = json!({ + "filterableAttributes": ["genre", "year"], + "sortableAttributes": ["year"] + }); + let (task, code) = index.update_settings(settings).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search before rename + let search_params = json!({ + "q": "matrix", + "filter": "genre = 'Sci-Fi'", + "sort": ["year:asc"] + }); + let (results_before, code) = index.search_post(search_params.clone()).await; + assert_eq!(code, 200); + assert_eq!(results_before["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_before["hits"][0]["title"], "The Matrix"); + + // Rename the index + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Search after rename + let (results_after, code) = + server.service.post(format!("/indexes/{}/search", new_uid), search_params).await; + assert_eq!(code, 200); + assert_eq!(results_after["hits"].as_array().unwrap().len(), 1); + assert_eq!(results_after["hits"][0]["title"], "The Matrix"); + + // Verify facet search also works + let facet_search = json!({ + "facetQuery": "Sci", + "facetName": "genre" + }); + let (facet_results, code) = + server.service.post(format!("/indexes/{}/facet-search", new_uid), facet_search).await; + assert_eq!(code, 200); + assert_eq!(facet_results["facetHits"].as_array().unwrap().len(), 1); + assert_eq!(facet_results["facetHits"][0]["value"], "Sci-Fi"); +} + +#[actix_rt::test] +async fn rename_index_with_pending_tasks() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // Create index + let (task, code) = index.create(None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Add initial documents + let documents = json!([ + { "id": 1, "title": "Document 1" } + ]); + let (task, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Start a rename + let new_uid = format!("{}_renamed", index.uid); + let body = json!({ "uid": &new_uid }); + let (rename_task, code) = index.service.patch(format!("/indexes/{}", index.uid), body).await; + assert_eq!(code, 202); + + // Try to add documents to the old index while rename is pending + let more_documents = json!([ + { "id": 2, "title": "Document 2" } + ]); + let (_, code) = index.add_documents(more_documents, None).await; + assert_eq!(code, 202); + + // Wait for rename to complete + server.wait_task(rename_task.uid()).await.succeeded(); + + // Add documents to the new index + let final_documents = json!([ + { "id": 3, "title": "Document 3" } + ]); + let (task, code) = + server.service.post(format!("/indexes/{}/documents", new_uid), final_documents).await; + assert_eq!(code, 202); + server.wait_task(task.uid()).await.succeeded(); + + // Verify all documents are accessible + let (response, code) = server.service.get(format!("/indexes/{}/documents", new_uid)).await; + assert_eq!(code, 200); + let docs = response["results"].as_array().unwrap(); + assert!(!docs.is_empty()); // At least the initial document should be there +} diff --git a/crates/meilisearch/tests/index/update_index.rs b/crates/meilisearch/tests/index/update_index.rs index 262324bcf..5b72d4328 100644 --- a/crates/meilisearch/tests/index/update_index.rs +++ b/crates/meilisearch/tests/index/update_index.rs @@ -1,8 +1,12 @@ +use meili_snap::snapshot; + use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; use crate::common::encoder::Encoder; -use crate::common::{shared_does_not_exists_index, shared_index_with_documents, Server}; +use crate::common::{ + shared_does_not_exists_index, shared_empty_index, shared_index_with_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -106,3 +110,85 @@ async fn error_update_unexisting_index() { assert_eq!(response["error"], expected_response); } + +#[actix_rt::test] +async fn update_index_name() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + + let new_index = server.unique_index(); + let (task, _code) = index.update_raw(json!({ "uid": new_index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (response, code) = new_index.get().await; + snapshot!(code, @"200 OK"); + + assert_eq!(response["uid"], new_index.uid); + assert!(response.get("createdAt").is_some()); + assert!(response.get("updatedAt").is_some()); + + let created_at = + OffsetDateTime::parse(response["createdAt"].as_str().unwrap(), &Rfc3339).unwrap(); + let updated_at = + OffsetDateTime::parse(response["updatedAt"].as_str().unwrap(), &Rfc3339).unwrap(); + assert!(created_at < updated_at, "{created_at} should be inferior to {updated_at}"); + + snapshot!(response["primaryKey"], @"null"); + snapshot!(response.as_object().unwrap().len(), @"4"); +} + +#[actix_rt::test] +async fn update_index_name_to_itself() { + let server = Server::new_shared(); + let index = server.unique_index(); + let (task, _code) = index.create(None).await; + server.wait_task(task.uid()).await.succeeded(); + let (initial_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + let (task, _code) = index.update_raw(json!({ "uid": index.uid })).await; + server.wait_task(task.uid()).await.succeeded(); + + let (new_response, code) = index.get().await; + snapshot!(code, @"200 OK"); + + // Renaming an index to its own name should not change anything + assert_eq!(initial_response, new_response); +} + +#[actix_rt::test] +async fn error_update_index_name_to_already_existing_index() { + let server = Server::new_shared(); + let base_index = shared_empty_index().await; + let index = shared_index_with_documents().await; + + let (task, _status_code) = + index.update_raw_index_fail(json!({ "uid": base_index.uid }), server).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "SHARED_DOCUMENTS", + "status": "failed", + "type": "indexUpdate", + "canceledBy": null, + "details": { + "primaryKey": null, + "oldIndexUid": "SHARED_DOCUMENTS", + "newIndexUid": "EMPTY_INDEX" + }, + "error": { + "message": "Index `EMPTY_INDEX` already exists.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/network/mod.rs b/crates/meilisearch/tests/network/mod.rs index 60f73ed40..e25656eb0 100644 --- a/crates/meilisearch/tests/network/mod.rs +++ b/crates/meilisearch/tests/network/mod.rs @@ -46,7 +46,7 @@ async fn errors_on_param() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "Unknown field `selfie`: expected one of `remotes`, `self`", + "message": "Unknown field `selfie`: expected one of `remotes`, `self`, `sharding`", "code": "bad_request", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#bad_request" @@ -149,7 +149,7 @@ async fn errors_on_param() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { - "message": "Unknown field `doggo` inside `.remotes.new`: expected one of `url`, `searchApiKey`", + "message": "Unknown field `doggo` inside `.remotes.new`: expected one of `url`, `searchApiKey`, `writeApiKey`", "code": "invalid_network_remotes", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_network_remotes" @@ -192,9 +192,11 @@ async fn errors_on_param() { "remotes": { "kefir": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null } - } + }, + "sharding": false } "###); let (response, code) = server @@ -266,7 +268,8 @@ async fn auth() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "master", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -274,11 +277,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "master", - "remotes": {} -} -"###); + { + "self": "master", + "remotes": {}, + "sharding": false + } + "###); // try get with get permission server.use_api_key(get_network_key.as_str().unwrap()); @@ -286,11 +290,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "master", - "remotes": {} -} -"###); + { + "self": "master", + "remotes": {}, + "sharding": false + } + "###); // try update with update permission server.use_api_key(update_network_key.as_str().unwrap()); @@ -303,11 +308,12 @@ async fn auth() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" -{ - "self": "api_key", - "remotes": {} -} -"###); + { + "self": "api_key", + "remotes": {}, + "sharding": false + } + "###); // try with the other's permission let (response, code) = server.get_network().await; @@ -383,7 +389,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": null, - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -393,7 +400,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "myself", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -417,13 +425,16 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "foo" + "searchApiKey": "foo", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -443,13 +454,16 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -470,17 +484,21 @@ async fn get_and_set_network() { "remotes": { "myself": { "url": "http://localhost:7700", - "searchApiKey": null + "searchApiKey": null, + "writeApiKey": null }, "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -498,13 +516,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -518,13 +539,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -538,13 +562,16 @@ async fn get_and_set_network() { "remotes": { "them": { "url": "http://localhost:7702", - "searchApiKey": "baz" + "searchApiKey": "baz", + "writeApiKey": null }, "thy": { "url": "http://localhost:7701", - "searchApiKey": "bar" + "searchApiKey": "bar", + "writeApiKey": null } - } + }, + "sharding": false } "###); @@ -553,60 +580,69 @@ async fn get_and_set_network() { meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // still doing nothing let (response, code) = server.set_network(json!({"remotes": {}})).await; meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // good time to check GET let (response, code) = server.get_network().await; meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(meili_snap::json_string!(response), @r###" - { - "self": "thy", - "remotes": { - "them": { - "url": "http://localhost:7702", - "searchApiKey": "baz" - }, - "thy": { - "url": "http://localhost:7701", - "searchApiKey": "bar" - } - } + { + "self": "thy", + "remotes": { + "them": { + "url": "http://localhost:7702", + "searchApiKey": "baz", + "writeApiKey": null + }, + "thy": { + "url": "http://localhost:7701", + "searchApiKey": "bar", + "writeApiKey": null } - "###); + }, + "sharding": false + } + "###); // deleting everything let (response, code) = server @@ -619,7 +655,8 @@ async fn get_and_set_network() { meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "self": "thy", - "remotes": {} + "remotes": {}, + "sharding": false } "###); } diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 2b1623ff8..c435a491e 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -132,7 +132,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -140,7 +141,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; @@ -148,7 +150,8 @@ async fn remote_sharding() { snapshot!(json_string!(response), @r###" { "self": "ms2", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -444,7 +447,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -452,7 +456,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; @@ -460,7 +465,8 @@ async fn remote_sharding_retrieve_vectors() { snapshot!(json_string!(response), @r###" { "self": "ms2", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -934,7 +940,8 @@ async fn error_unregistered_remote() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -942,7 +949,8 @@ async fn error_unregistered_remote() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1052,7 +1060,8 @@ async fn error_no_weighted_score() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1060,7 +1069,8 @@ async fn error_no_weighted_score() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1185,7 +1195,8 @@ async fn error_bad_response() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1193,7 +1204,8 @@ async fn error_bad_response() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1322,7 +1334,8 @@ async fn error_bad_request() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1330,7 +1343,8 @@ async fn error_bad_request() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1452,7 +1466,8 @@ async fn error_bad_request_facets_by_index() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1460,7 +1475,8 @@ async fn error_bad_request_facets_by_index() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1593,7 +1609,8 @@ async fn error_bad_request_facets_by_index_facet() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1601,7 +1618,8 @@ async fn error_bad_request_facets_by_index_facet() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1743,7 +1761,8 @@ async fn error_remote_does_not_answer() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1751,7 +1770,8 @@ async fn error_remote_does_not_answer() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -1944,7 +1964,8 @@ async fn error_remote_404() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -1952,7 +1973,8 @@ async fn error_remote_404() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2139,7 +2161,8 @@ async fn error_remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2147,7 +2170,8 @@ async fn error_remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2299,7 +2323,8 @@ async fn remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2307,7 +2332,8 @@ async fn remote_sharding_auth() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2454,7 +2480,8 @@ async fn error_remote_500() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2462,7 +2489,8 @@ async fn error_remote_500() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); @@ -2633,7 +2661,8 @@ async fn error_remote_500_once() { snapshot!(json_string!(response), @r###" { "self": "ms0", - "remotes": {} + "remotes": {}, + "sharding": false } "###); let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; @@ -2641,7 +2670,8 @@ async fn error_remote_500_once() { snapshot!(json_string!(response), @r###" { "self": "ms1", - "remotes": {} + "remotes": {}, + "sharding": false } "###); diff --git a/crates/meilisearch/tests/swap_indexes/errors.rs b/crates/meilisearch/tests/swap_indexes/errors.rs index d136dfc0f..ee992372e 100644 --- a/crates/meilisearch/tests/swap_indexes/errors.rs +++ b/crates/meilisearch/tests/swap_indexes/errors.rs @@ -1,6 +1,9 @@ use meili_snap::*; -use crate::common::Server; +use crate::common::{ + shared_empty_index, shared_index_with_documents, shared_index_with_geo_documents, + shared_index_with_nested_documents, Server, +}; use crate::json; #[actix_rt::test] @@ -92,3 +95,112 @@ async fn swap_indexes_bad_indexes() { } "###); } + +#[actix_rt::test] +async fn swap_indexes_bad_rename() { + let server = Server::new_shared(); + + let (response, code) = + server.index_swap(json!([{ "indexes": ["kefir", "intel"], "rename": "hello" }])).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r#" + { + "message": "Invalid value type at `[0].rename`: expected a boolean, but found a string: `\"hello\"`", + "code": "invalid_swap_rename", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_swap_rename" + } + "#); +} + +#[actix_rt::test] +async fn swap_indexes_rename_to_already_existing_index() { + let server = Server::new_shared(); + let already_existing_index = shared_empty_index().await; + let base_index = shared_index_with_documents().await; + + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + } + ] + }, + "error": { + "message": "Cannot rename `SHARED_DOCUMENTS` to `EMPTY_INDEX` as the index already exists. Hint: You can remove `EMPTY_INDEX` first and then do your remove.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let base_index_2 = shared_index_with_geo_documents().await; + let already_existing_index_2 = shared_index_with_nested_documents().await; + let (response, _code) = server + .index_swap( + json!([{ "indexes": [base_index.uid, already_existing_index.uid], "rename": true }, { "indexes": [base_index_2.uid, already_existing_index_2.uid], "rename": true }]), + ) + .await; + let response = server.wait_task(response.uid()).await; + snapshot!(response, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": null, + "status": "failed", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "SHARED_DOCUMENTS", + "EMPTY_INDEX" + ], + "rename": true + }, + { + "indexes": [ + "SHARED_GEO_DOCUMENTS", + "SHARED_NESTED_DOCUMENTS" + ], + "rename": true + } + ] + }, + "error": { + "message": "The following indexes are being renamed but cannot because their new name conflicts with an already existing index: `EMPTY_INDEX`, `SHARED_NESTED_DOCUMENTS`. Renaming doesn't overwrite the other index name.", + "code": "index_already_exists", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#index_already_exists" + }, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); +} diff --git a/crates/meilisearch/tests/swap_indexes/mod.rs b/crates/meilisearch/tests/swap_indexes/mod.rs index bf84d5823..edb7fab49 100644 --- a/crates/meilisearch/tests/swap_indexes/mod.rs +++ b/crates/meilisearch/tests/swap_indexes/mod.rs @@ -73,7 +73,7 @@ async fn swap_indexes() { snapshot!(code, @"200 OK"); // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -89,7 +89,8 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false } ] }, @@ -102,7 +103,7 @@ async fn swap_indexes() { { "uid": 1, "batchUid": 1, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -139,7 +140,7 @@ async fn swap_indexes() { "from": 2, "next": null } - "###); + "#); // BUT, the data in `a` should now points to the data that was in `b`. // And the opposite is true as well @@ -228,7 +229,7 @@ async fn swap_indexes() { // 2. stays unchanged // 3. now have the indexUid `d` instead of `c` // 4. now have the indexUid `c` instead of `d` - snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r###" + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" { "results": [ { @@ -244,13 +245,15 @@ async fn swap_indexes() { "indexes": [ "a", "b" - ] + ], + "rename": false }, { "indexes": [ "c", "d" - ] + ], + "rename": false } ] }, @@ -263,7 +266,7 @@ async fn swap_indexes() { { "uid": 4, "batchUid": 4, - "indexUid": "c", + "indexUid": "d", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -307,7 +310,8 @@ async fn swap_indexes() { "indexes": [ "b", "a" - ] + ], + "rename": false } ] }, @@ -337,7 +341,7 @@ async fn swap_indexes() { { "uid": 0, "batchUid": 0, - "indexUid": "a", + "indexUid": "b", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -357,7 +361,7 @@ async fn swap_indexes() { "from": 5, "next": null } - "###); + "#); // - The data in `a` should point to `a`. // - The data in `b` should point to `b`. @@ -372,3 +376,255 @@ async fn swap_indexes() { let (res, _) = d.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(res["results"], @r###"[{"id":1,"index":"c"}]"###); } + +#[actix_rt::test] +async fn swap_rename_indexes() { + let server = Server::new().await; + let a = server.index("a"); + let b = server.index("b"); + a.create(None).await; + a.add_documents(json!({ "id": 1, "index": "a"}), None).await; + + let (res, _code) = server.index_swap(json!([{ "indexes": ["a", "b"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + let (tasks, _code) = server.tasks().await; + + // Notice how the task 0 which was initially representing the creation of the index `A` now represents the creation of the index `B`. + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 2, + "batchUid": 2, + "indexUid": null, + "status": "succeeded", + "type": "indexSwap", + "canceledBy": null, + "details": { + "swaps": [ + { + "indexes": [ + "a", + "b" + ], + "rename": true + } + ] + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 2, + "next": null + } + "#); + + // BUT, `a` should not exists + let (res, code) = a.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"404 Not Found"); + snapshot!(res["results"], @"null"); + + // And its data should be in b + let (res, code) = b.get_all_documents(GetAllDocumentsOptions::default()).await; + snapshot!(code, @"200 OK"); + snapshot!(res["results"], @r#"[{"id":1,"index":"a"}]"#); + + // No tasks should be linked to the index a + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 1, + "limit": 20, + "from": null, + "next": null + } + "#); + + // They should be linked to the index b + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 1, + "batchUid": 1, + "indexUid": "b", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "b", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 3, + "limit": 20, + "from": 1, + "next": null + } + "#); + + // ===== Now, we can delete the index `b`, but its tasks will stays + // if we then make a new `b` index and rename it to be called `a` + // the tasks currently available in `b` should not be moved + + let (res, _) = b.delete().await; + server.wait_task(res.uid()).await.succeeded(); + + b.create(Some("kefir")).await; + let (res, _code) = server.index_swap(json!([{ "indexes": ["b", "a"], "rename": true }])).await; + server.wait_task(res.uid()).await.succeeded(); + + // `a` now contains everything + let (tasks, _code) = server.tasks_filter("indexUids=a").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [ + { + "uid": 4, + "batchUid": 4, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": "kefir" + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 3, + "batchUid": 3, + "indexUid": "a", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 1, + "batchUid": 1, + "indexUid": "a", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + }, + { + "uid": 0, + "batchUid": 0, + "indexUid": "a", + "status": "succeeded", + "type": "indexCreation", + "canceledBy": null, + "details": { + "primaryKey": null + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + ], + "total": 6, + "limit": 20, + "from": 4, + "next": null + } + "#); + + // And `b` is empty + let (tasks, _code) = server.tasks_filter("indexUids=b").await; + snapshot!(json_string!(tasks, { ".results[].duration" => "[duration]", ".results[].enqueuedAt" => "[date]", ".results[].startedAt" => "[date]", ".results[].finishedAt" => "[date]" }), @r#" + { + "results": [], + "total": 2, + "limit": 20, + "from": null, + "next": null + } + "#); +} diff --git a/crates/meilisearch/tests/tasks/mod.rs b/crates/meilisearch/tests/tasks/mod.rs index 09700d3c5..6397ad6ad 100644 --- a/crates/meilisearch/tests/tasks/mod.rs +++ b/crates/meilisearch/tests/tasks/mod.rs @@ -895,7 +895,7 @@ async fn test_summarized_index_swap() { server.wait_task(task.uid()).await.failed(); let (task, _) = server.get_task(task.uid()).await; snapshot!(task, - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -909,7 +909,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -924,7 +925,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); let doggos_index = server.unique_index(); let (task, _code) = doggos_index.create(None).await; @@ -941,7 +942,7 @@ async fn test_summarized_index_swap() { let (task, _) = server.get_task(task.uid()).await; snapshot!(json_string!(task, { ".uid" => "[uid]", ".batchUid" => "[batch_uid]", ".**.indexes[0]" => "doggos", ".**.indexes[1]" => "cattos", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }), - @r###" + @r#" { "uid": "[uid]", "batchUid": "[batch_uid]", @@ -955,7 +956,8 @@ async fn test_summarized_index_swap() { "indexes": [ "doggos", "cattos" - ] + ], + "rename": false } ] }, @@ -965,7 +967,7 @@ async fn test_summarized_index_swap() { "startedAt": "[date]", "finishedAt": "[date]" } - "###); + "#); } #[actix_web::test] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index 5d120ba2f..831569ffd 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.1"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.18.0"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.17.2 is higher than the Meilisearch version 1.17.1. Downgrade is not supported"); + snapshot!(err, @"Database version 1.18.1 is higher than the Meilisearch version 1.18.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index e7d8768be..0567e8f2c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 61dd95786..aca994428 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 8103ceed2..95b23e253 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 81259377c..96a3af5a2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.1" + "upgradeTo": "v1.18.0" }, "error": null, "duration": "[duration]", diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index b7bcd8c68..b3245978b 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -115,6 +115,7 @@ utoipa = { version = "5.4.0", features = [ lru = "0.14.0" geo-types = "0.7.16" zerometry = "0.1.0" +twox-hash = { version = "2.1.1", default-features = false, features = ["std", "xxhash3_64", "xxhash64"] } [dev-dependencies] mimalloc = { version = "0.1.47", default-features = false } diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index ba5a5bc10..94868215f 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1471,7 +1471,7 @@ impl Index { .0) } - pub(crate) fn set_updated_at( + pub fn set_updated_at( &self, wtxn: &mut RwTxn<'_>, time: &time::OffsetDateTime, diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index 6b8c25ab8..60e471579 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -76,6 +76,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6e34961e7..b831d55cc 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -84,6 +84,7 @@ impl TempIndex { &mut new_fields_ids_map, &|| false, Progress::default(), + None, )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -167,6 +168,7 @@ impl TempIndex { &mut new_fields_ids_map, &|| false, Progress::default(), + None, )?; if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { @@ -242,6 +244,7 @@ fn aborting_indexation() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 58d8fdf3e..966f2d6f1 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -1979,6 +1979,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2031,6 +2032,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2119,6 +2121,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2308,6 +2311,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2371,6 +2375,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2425,6 +2430,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2478,6 +2484,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2533,6 +2540,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2593,6 +2601,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2646,6 +2655,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2699,6 +2709,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2910,6 +2921,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -2970,6 +2982,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); @@ -3027,6 +3040,7 @@ mod tests { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/src/update/new/indexer/document_operation.rs b/crates/milli/src/update/new/indexer/document_operation.rs index 98faaf145..4e3725c6e 100644 --- a/crates/milli/src/update/new/indexer/document_operation.rs +++ b/crates/milli/src/update/new/indexer/document_operation.rs @@ -17,6 +17,7 @@ use super::guess_primary_key::retrieve_or_guess_primary_key; use crate::documents::PrimaryKey; use crate::progress::{AtomicPayloadStep, Progress}; use crate::update::new::document::{DocumentContext, Versions}; +use crate::update::new::indexer::enterprise_edition::sharding::Shards; use crate::update::new::steps::IndexingStep; use crate::update::new::thread_local::MostlySend; use crate::update::new::{DocumentIdentifiers, Insertion, Update}; @@ -71,6 +72,7 @@ impl<'pl> DocumentOperation<'pl> { new_fields_ids_map: &mut FieldsIdsMap, must_stop_processing: &MSP, progress: Progress, + shards: Option<&Shards>, ) -> Result<(DocumentOperationChanges<'pl>, Vec, Option>)> where MSP: Fn() -> bool, @@ -107,6 +109,7 @@ impl<'pl> DocumentOperation<'pl> { &mut bytes, &docids_version_offsets, IndexDocumentsMethod::ReplaceDocuments, + shards, payload, ), Payload::Update(payload) => extract_addition_payload_changes( @@ -120,6 +123,7 @@ impl<'pl> DocumentOperation<'pl> { &mut bytes, &docids_version_offsets, IndexDocumentsMethod::UpdateDocuments, + shards, payload, ), Payload::Deletion(to_delete) => extract_deletion_payload_changes( @@ -127,6 +131,7 @@ impl<'pl> DocumentOperation<'pl> { rtxn, &mut available_docids, &docids_version_offsets, + shards, to_delete, ), }; @@ -173,6 +178,7 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( bytes: &mut u64, main_docids_version_offsets: &hashbrown::HashMap<&'pl str, PayloadOperations<'pl>>, method: IndexDocumentsMethod, + shards: Option<&Shards>, payload: &'pl [u8], ) -> Result>> { use IndexDocumentsMethod::{ReplaceDocuments, UpdateDocuments}; @@ -210,12 +216,20 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( primary_key.as_ref().unwrap() }; + let current_offset = iter.byte_offset(); + let content = &payload[previous_offset..current_offset]; + previous_offset = current_offset; + let external_id = retrieved_primary_key.extract_fields_and_docid(doc, new_fields_ids_map, indexer)?; let external_id = external_id.to_de(); - let current_offset = iter.byte_offset(); - let document_offset = DocumentOffset { content: &payload[previous_offset..current_offset] }; + + if shards.is_some_and(|shards| !shards.must_process(external_id)) { + continue; + } + + let document_offset = DocumentOffset { content }; match main_docids_version_offsets.get(external_id) { None => { @@ -299,8 +313,6 @@ fn extract_addition_payload_changes<'r, 'pl: 'r>( }, }, } - - previous_offset = iter.byte_offset(); } if payload.is_empty() { @@ -329,11 +341,16 @@ fn extract_deletion_payload_changes<'s, 'pl: 's>( rtxn: &RoTxn, available_docids: &mut AvailableIds, main_docids_version_offsets: &hashbrown::HashMap<&'s str, PayloadOperations<'pl>>, + shards: Option<&Shards>, to_delete: &'pl [&'pl str], ) -> Result>> { let mut new_docids_version_offsets = hashbrown::HashMap::<&str, PayloadOperations<'pl>>::new(); for external_id in to_delete { + if shards.is_some_and(|shards| !shards.must_process(external_id)) { + continue; + } + match main_docids_version_offsets.get(external_id) { None => { match index.external_documents_ids().get(rtxn, external_id) { diff --git a/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs b/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs new file mode 100644 index 000000000..9f89e8e31 --- /dev/null +++ b/crates/milli/src/update/new/indexer/enterprise_edition/mod.rs @@ -0,0 +1,6 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +pub mod sharding; diff --git a/crates/milli/src/update/new/indexer/enterprise_edition/sharding.rs b/crates/milli/src/update/new/indexer/enterprise_edition/sharding.rs new file mode 100644 index 000000000..c0322f868 --- /dev/null +++ b/crates/milli/src/update/new/indexer/enterprise_edition/sharding.rs @@ -0,0 +1,22 @@ +// Copyright © 2025 Meilisearch Some Rights Reserved +// This file is part of Meilisearch Enterprise Edition (EE). +// Use of this source code is governed by the Business Source License 1.1, +// as found in the LICENSE-EE file or at + +use std::hash::{BuildHasher as _, BuildHasherDefault}; + +pub struct Shards { + pub own: Vec, + pub others: Vec, +} + +impl Shards { + pub fn must_process(&self, docid: &str) -> bool { + let hasher = BuildHasherDefault::::new(); + let to_hash = |shard: &String| hasher.hash_one((shard, docid)); + + let max_hash = self.others.iter().map(to_hash).max().unwrap_or_default(); + + self.own.iter().map(to_hash).any(|hash| hash > max_hash) + } +} diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index e66b0e01e..dbe51250d 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -31,6 +31,7 @@ pub(crate) mod de; pub mod document_changes; mod document_deletion; mod document_operation; +pub mod enterprise_edition; mod extract; mod guess_primary_key; mod partial_dump; diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index ecd1cec6c..718dd41f7 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -8,7 +8,6 @@ use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_15::Latest_V1_14_To_Latest_V1_15; -use v1_16::Latest_V1_16_To_V1_17_0; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; @@ -35,7 +34,6 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, - &Latest_V1_16_To_V1_17_0 {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -64,7 +62,8 @@ const fn start(from: (u32, u32, u32)) -> Option { // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), (1, 16, _) => function_index!(7), - (1, 17, _) => function_index!(8), + (1, 17, _) => function_index!(7), + (1, 18, _) => function_index!(7), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs index 02dd136ce..f43efd77d 100644 --- a/crates/milli/src/update/upgrade/v1_16.rs +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -46,22 +46,3 @@ impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { (1, 16, 0) } } - -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_16_To_V1_17_0(); - -impl UpgradeIndex for Latest_V1_16_To_V1_17_0 { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - (1, 17, 0) - } -} diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index cc1b85369..2f347516b 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -59,6 +59,7 @@ fn test_facet_distribution_with_no_facet_values() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index 578a22009..f88f6c392 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -97,6 +97,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 3f8134085..f92cf9e51 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -329,6 +329,7 @@ fn criteria_ascdesc() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap(); diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index 95ff85165..d44848353 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -138,6 +138,7 @@ fn test_typo_disabled_on_word() { &mut new_fields_ids_map, &|| false, Progress::default(), + None, ) .unwrap();