From 3097314b9d4b0429547b85d41d2412d0bb451709 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Dec 2025 21:43:40 +0100 Subject: [PATCH 1/5] Make snapshots independent on the version --- crates/index-scheduler/src/insta_snapshot.rs | 24 ++++++++++++++--- crates/meilisearch/tests/upgrade/mod.rs | 28 ++++++++++++++++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 2f554adf4..ac633aac4 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -6,7 +6,7 @@ use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str}; use meilisearch_types::heed::{Database, RoTxn}; use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32}; use meilisearch_types::tasks::{Details, Kind, Status, Task}; -use meilisearch_types::versioning; +use meilisearch_types::versioning::{self, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use roaring::RoaringBitmap; use crate::index_mapper::IndexMapper; @@ -320,7 +320,11 @@ fn snapshot_details(d: &Details) -> String { format!("{{ url: {url:?}, api_key: {api_key:?}, payload_size: {payload_size:?}, indexes: {indexes:?} }}") } Details::UpgradeDatabase { from, to } => { - format!("{{ from: {from:?}, to: {to:?} }}") + if to == &(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) { + format!("{{ from: {from:?}, to: [current version] }}") + } else { + format!("{{ from: {from:?}, to: {to:?} }}") + } } Details::IndexCompaction { index_uid, pre_compaction_size, post_compaction_size } => { format!("{{ index_uid: {index_uid:?}, pre_compaction_size: {pre_compaction_size:?}, post_compaction_size: {post_compaction_size:?} }}") @@ -400,7 +404,21 @@ pub fn snapshot_batch(batch: &Batch) -> String { snap.push('{'); snap.push_str(&format!("uid: {uid}, ")); - snap.push_str(&format!("details: {}, ", serde_json::to_string(details).unwrap())); + let details = if let Some(upgrade_to) = &details.upgrade_to { + if upgrade_to.as_str() + == format!("v{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}").as_str() + { + let mut details = details.clone(); + + details.upgrade_to = Some("[current version]".into()); + serde_json::to_string(&details).unwrap() + } else { + serde_json::to_string(details).unwrap() + } + } else { + serde_json::to_string(details).unwrap() + }; + snap.push_str(&format!("details: {details}, ")); snap.push_str(&format!("stats: {}, ", serde_json::to_string(&stats).unwrap())); if !embedder_stats.skip_serializing() { snap.push_str(&format!( diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index a096ff4af..65e82a197 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -42,8 +42,16 @@ async fn version_too_old() { std::fs::create_dir_all(&db_path).unwrap(); std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; - let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.28.2"); + let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err().to_string(); + + let major = meilisearch_types::versioning::VERSION_MAJOR; + let minor = meilisearch_types::versioning::VERSION_MINOR; + let patch = meilisearch_types::versioning::VERSION_PATCH; + + let current_version = format!("{major}.{minor}.{patch}"); + let err = err.replace(¤t_version, "[current version]"); + + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v[current version]"); } #[actix_rt::test] @@ -54,11 +62,21 @@ async fn version_requires_downgrade() { std::fs::create_dir_all(&db_path).unwrap(); let major = meilisearch_types::versioning::VERSION_MAJOR; let minor = meilisearch_types::versioning::VERSION_MINOR; - let patch = meilisearch_types::versioning::VERSION_PATCH + 1; - std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); + let mut patch = meilisearch_types::versioning::VERSION_PATCH; + + let current_version = format!("{major}.{minor}.{patch}"); + patch += 1; + let future_version = format!("{major}.{minor}.{patch}"); + + std::fs::write(db_path.join("VERSION"), &future_version).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.28.3 is higher than the Meilisearch version 1.28.2. Downgrade is not supported"); + + let err = err.to_string(); + let err = err.replace(¤t_version, "[current version]"); + let err = err.replace(&future_version, "[future version]"); + + snapshot!(err, @"Database version [future version] is higher than the Meilisearch version [current version]. Downgrade is not supported"); } #[actix_rt::test] From c06ffb31d1d6feb0851b0637ce6eed619cae34ab Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Dec 2025 21:44:51 +0100 Subject: [PATCH 2/5] Update snapshots --- .../after_processing_everything.snap | 4 +- .../register_automatic_upgrade_task.snap | 2 +- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 4 +- .../upgrade_task_failed_again.snap | 4 +- .../upgrade_task_succeeded.snap | 4 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- .../tests/upgrade/v1_12/v1_12_0.rs | 44 +++++++++---------- 15 files changed, 40 insertions(+), 40 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index e8f7af82e..3253f3f6c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index 2a5dd37fd..6a36e4800 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 9204589d5..6ebbdaca4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 23ba57b8c..a85198ad8 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index fb90eea8e..4038732a7 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.2"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index a8cd5b692..982732f74 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 28, 2) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.2"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 6bb4e76a6..8e8a2dfcf 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 6bb4e76a6..8e8a2dfcf 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 6bb4e76a6..8e8a2dfcf 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index d1f6cdb7a..84ef227c9 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index d1f6cdb7a..84ef227c9 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index d1f6cdb7a..84ef227c9 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index e56b63767..7ee2445f3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 306582d13..d4cd9ab22 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.28.2" + "upgradeTo": "[current version]" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs index 2234f2895..4706eae82 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -166,55 +166,55 @@ async fn check_the_index_scheduler(server: &Server) { // We rewrite the first task for all calls because it may be the upgrade database with unknown dates and duration. // The other tasks should NOT change let (tasks, _) = server.tasks_filter("limit=1000").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed"); let (batches, _) = server.batches_filter("limit=1000").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed"); // Tests all the tasks query parameters let (tasks, _) = server.tasks_filter("uids=10").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_uids_equal_10"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_uids_equal_10"); let (tasks, _) = server.tasks_filter("batchUids=10").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_batchUids_equal_10"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_batchUids_equal_10"); let (tasks, _) = server.tasks_filter("statuses=canceled").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_statuses_equal_canceled"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_statuses_equal_canceled"); // types has already been tested above to retrieve the upgrade database let (tasks, _) = server.tasks_filter("canceledBy=19").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_canceledBy_equal_19"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_canceledBy_equal_19"); let (tasks, _) = server.tasks_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); let (tasks, _) = server.tasks_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); let (tasks, _) = server.tasks_filter("beforeStartedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); let (tasks, _) = server.tasks_filter("afterStartedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41"); let (tasks, _) = server.tasks_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); let (tasks, _) = server.tasks_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); // Tests all the batches query parameters let (batches, _) = server.batches_filter("uids=10").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10"); let (batches, _) = server.batches_filter("batchUids=10").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10"); let (batches, _) = server.batches_filter("statuses=canceled").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled"); // types has already been tested above to retrieve the upgrade database let (batches, _) = server.batches_filter("canceledBy=19").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19"); let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41"); let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41"); let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41"); let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41"); let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41"); let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await; - snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); + snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41"); let (stats, _) = server.stats().await; assert_json_snapshot!(stats, { From c0e31a4f01882e8de93b25d0bd137d73781c7aeb Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Dec 2025 22:47:29 +0100 Subject: [PATCH 3/5] Switch to migration-oriented dumpless upgrade --- crates/index-scheduler/src/upgrade/mod.rs | 157 ++++++++---------- crates/index-scheduler/src/versioning.rs | 9 +- crates/milli/src/update/upgrade/mod.rs | 186 +++++----------------- crates/milli/src/update/upgrade/v1_12.rs | 42 +++-- crates/milli/src/update/upgrade/v1_13.rs | 38 +---- crates/milli/src/update/upgrade/v1_14.rs | 21 ++- crates/milli/src/update/upgrade/v1_15.rs | 20 +-- crates/milli/src/update/upgrade/v1_16.rs | 13 +- 8 files changed, 157 insertions(+), 329 deletions(-) diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index e338e1633..3d2372221 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -1,7 +1,7 @@ use anyhow::bail; use meilisearch_types::heed::{Env, RwTxn, WithoutTls}; use meilisearch_types::tasks::{Details, KindWithContent, Status, Task}; -use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; +use meilisearch_types::versioning; use time::OffsetDateTime; use tracing::info; @@ -9,83 +9,82 @@ use crate::queue::TaskQueue; use crate::versioning::Versioning; trait UpgradeIndexScheduler { - fn upgrade( - &self, - env: &Env, - wtxn: &mut RwTxn, - original: (u32, u32, u32), - ) -> anyhow::Result<()>; - fn target_version(&self) -> (u32, u32, u32); + fn upgrade(&self, env: &Env, wtxn: &mut RwTxn) -> anyhow::Result<()>; + /// Whether the migration should be applied, depending on the initial version of the index scheduler before + /// any migration was applied + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool; + /// A progress-centric description of the migration + fn description(&self) -> &'static str; } +/// Upgrade the index scheduler to the binary version. +/// +/// # Warning +/// +/// The current implementation uses a single wtxn to the index scheduler for the whole duration of the upgrade. +/// If migrations start taking take a long time, it might prevent tasks from being registered. +/// If this issue manifests, then it can be mitigated by adding a `fn target_version` to `UpgradeIndexScheduler`, +/// to be able to write intermediate versions and drop the wtxn between applying migrations. pub fn upgrade_index_scheduler( env: &Env, versioning: &Versioning, - from: (u32, u32, u32), - to: (u32, u32, u32), + initial_version: (u32, u32, u32), ) -> anyhow::Result<()> { - let current_major = to.0; - let current_minor = to.1; - let current_patch = to.2; + let target_major: u32 = versioning::VERSION_MAJOR; + let target_minor: u32 = versioning::VERSION_MINOR; + let target_patch: u32 = versioning::VERSION_PATCH; + let target_version = (target_major, target_minor, target_patch); - let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[ - // This is the last upgrade function, it will be called when the index is up to date. - // any other upgrade function should be added before this one. - &ToCurrentNoOp {}, - ]; - - let start = match from { - (1, 12, _) => 0, - (1, 13, _) => 0, - (1, 14, _) => 0, - (1, 15, _) => 0, - (1, 16, _) => 0, - (1, 17, _) => 0, - (1, 18, _) => 0, - (1, 19, _) => 0, - (1, 20, _) => 0, - (1, 21, _) => 0, - (1, 22, _) => 0, - (1, 23, _) => 0, - (1, 24, _) => 0, - (1, 25, _) => 0, - (1, 26, _) => 0, - (1, 27, _) => 0, - (1, 28, _) => 0, - (major, minor, patch) => { - if major > current_major - || (major == current_major && minor > current_minor) - || (major == current_major && minor == current_minor && patch > current_patch) - { - bail!( - "Database version {major}.{minor}.{patch} is higher than the Meilisearch version {current_major}.{current_minor}.{current_patch}. Downgrade is not supported", - ); - } else if major < 1 || (major == current_major && minor < 12) { - bail!( - "Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{current_major}.{current_minor}.{current_patch}", - ); - } else { - bail!("Unknown database version: v{major}.{minor}.{patch}"); - } - } - }; - - info!("Upgrading the task queue"); - let mut local_from = from; - for upgrade in upgrade_functions[start..].iter() { - let target = upgrade.target_version(); - info!( - "Upgrading from v{}.{}.{} to v{}.{}.{}", - local_from.0, local_from.1, local_from.2, target.0, target.1, target.2 - ); - let mut wtxn = env.write_txn()?; - upgrade.upgrade(env, &mut wtxn, local_from)?; - versioning.set_version(&mut wtxn, target)?; - wtxn.commit()?; - local_from = target; + if initial_version == target_version { + return Ok(()); } + let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[ + // List all upgrade functions to apply in order here. + ]; + + let (initial_major, initial_minor, initial_patch) = initial_version; + + if initial_version > target_version { + bail!( + "Database version {initial_major}.{initial_minor}.{initial_patch} is higher than the Meilisearch version {target_major}.{target_minor}.{target_patch}. Downgrade is not supported", + ); + } + + if initial_version < (1, 12, 0) { + bail!( + "Database version {initial_major}.{initial_minor}.{initial_patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{initial_major}.{initial_minor}.{initial_patch} and import it in the v{target_major}.{target_minor}.{target_patch}", + ); + } + + info!("Upgrading the task queue"); let mut wtxn = env.write_txn()?; + let migration_count = upgrade_functions.len(); + for (migration_index, upgrade) in upgrade_functions.iter().enumerate() { + if upgrade.must_upgrade(initial_version) { + info!( + "[{migration_index}/{migration_count}]Applying migration: {}", + upgrade.description() + ); + + upgrade.upgrade(env, &mut wtxn)?; + + info!( + "[{}/{migration_count}]Migration applied: {}", + migration_index + 1, + upgrade.description() + ) + } else { + info!( + "[{migration_index}/{migration_count}]Skipping unnecessary migration: {}", + upgrade.description() + ) + } + } + + versioning.set_version(&mut wtxn, target_version)?; + info!("Task queue upgraded, spawning the upgrade database task"); + let queue = TaskQueue::new(env, &mut wtxn)?; let uid = queue.next_task_id(&wtxn)?; queue.register( @@ -98,9 +97,9 @@ pub fn upgrade_index_scheduler( finished_at: None, error: None, canceled_by: None, - details: Some(Details::UpgradeDatabase { from, to }), + details: Some(Details::UpgradeDatabase { from: initial_version, to: target_version }), status: Status::Enqueued, - kind: KindWithContent::UpgradeDatabase { from }, + kind: KindWithContent::UpgradeDatabase { from: initial_version }, network: None, custom_metadata: None, }, @@ -109,21 +108,3 @@ pub fn upgrade_index_scheduler( Ok(()) } - -#[allow(non_camel_case_types)] -struct ToCurrentNoOp {} - -impl UpgradeIndexScheduler for ToCurrentNoOp { - fn upgrade( - &self, - _env: &Env, - _wtxn: &mut RwTxn, - _original: (u32, u32, u32), - ) -> anyhow::Result<()> { - Ok(()) - } - - fn target_version(&self) -> (u32, u32, u32) { - (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) - } -} diff --git a/crates/index-scheduler/src/versioning.rs b/crates/index-scheduler/src/versioning.rs index b0cb7fdb5..c03b814dc 100644 --- a/crates/index-scheduler/src/versioning.rs +++ b/crates/index-scheduler/src/versioning.rs @@ -64,14 +64,7 @@ impl Versioning { }; wtxn.commit()?; - let bin_major: u32 = versioning::VERSION_MAJOR; - let bin_minor: u32 = versioning::VERSION_MINOR; - let bin_patch: u32 = versioning::VERSION_PATCH; - let to = (bin_major, bin_minor, bin_patch); - - if from != to { - upgrade_index_scheduler(env, &this, from, to)?; - } + upgrade_index_scheduler(env, &this, from)?; // Once we reach this point it means the upgrade process, if there was one is entirely finished // we can safely say we reached the latest version of the index scheduler diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index b9160f8ba..e44ae79c7 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -5,103 +5,36 @@ mod v1_15; mod v1_16; use heed::RwTxn; -use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; -use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; -use v1_14::Latest_V1_13_To_Latest_V1_14; -use v1_15::Latest_V1_14_To_Latest_V1_15; -use v1_16::Latest_V1_15_To_V1_16_0; +use v1_12::{FixFieldDistribution, RecomputeStats}; +use v1_13::AddNewStats; +use v1_14::UpgradeArroyVersion; +use v1_15::RecomputeWordFst; +use v1_16::SwitchToMultimodal; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; use crate::{Index, InternalError, Result}; trait UpgradeIndex { + /// Returns `true` if `upgrade` should be called when the index started with version `initial_version`. + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool; + /// Returns `true` if the index scheduler must regenerate its cached stats. - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - original: (u32, u32, u32), - progress: Progress, - ) -> Result; - fn target_version(&self) -> (u32, u32, u32); + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result; + + /// Description of the upgrade for progress display purposes. + fn description(&self) -> &'static str; } const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ - &V1_12_To_V1_12_3 {}, - &V1_12_3_To_V1_13_0 {}, - &V1_13_0_To_V1_13_1 {}, - &V1_13_1_To_Latest_V1_13 {}, - &Latest_V1_13_To_Latest_V1_14 {}, - &Latest_V1_14_To_Latest_V1_15 {}, - &Latest_V1_15_To_V1_16_0 {}, - &ToTargetNoOp { target: (1, 18, 0) }, - &ToTargetNoOp { target: (1, 19, 0) }, - &ToTargetNoOp { target: (1, 20, 0) }, - &ToTargetNoOp { target: (1, 21, 0) }, - &ToTargetNoOp { target: (1, 22, 0) }, - &ToTargetNoOp { target: (1, 23, 0) }, - &ToTargetNoOp { target: (1, 24, 0) }, - &ToTargetNoOp { target: (1, 25, 0) }, - &ToTargetNoOp { target: (1, 26, 0) }, - &ToTargetNoOp { target: (1, 27, 0) }, - &ToTargetNoOp { target: (1, 28, 0) }, - // This is the last upgrade function, it will be called when the index is up to date. - // any other upgrade function should be added before this one. - &ToCurrentNoOp {}, + &FixFieldDistribution {}, + &RecomputeStats {}, + &AddNewStats {}, + &UpgradeArroyVersion {}, + &RecomputeWordFst {}, + &SwitchToMultimodal {}, ]; -/// Causes a compile-time error if the argument is not in range of `0..UPGRADE_FUNCTIONS.len()` -macro_rules! function_index { - ($start:expr) => {{ - const _CHECK_INDEX: () = { - if $start >= $crate::update::upgrade::UPGRADE_FUNCTIONS.len() { - panic!("upgrade functions out of range") - } - }; - - $start - }}; -} - -const fn start(from: (u32, u32, u32)) -> Option { - let start = match from { - (1, 12, 0..=2) => function_index!(0), - (1, 12, 3..) => function_index!(1), - (1, 13, 0) => function_index!(2), - (1, 13, _) => function_index!(4), - (1, 14, _) => function_index!(5), - // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. - (1, 15, _) => function_index!(6), - (1, 16, _) | (1, 17, _) => function_index!(7), - (1, 18, _) => function_index!(8), - (1, 19, _) => function_index!(9), - (1, 20, _) => function_index!(10), - (1, 21, _) => function_index!(11), - (1, 22, _) => function_index!(12), - (1, 23, _) => function_index!(13), - (1, 24, _) => function_index!(14), - (1, 25, _) => function_index!(15), - (1, 26, _) => function_index!(16), - (1, 27, _) => function_index!(17), - (1, 28, _) => function_index!(18), - // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually - // considering dumpless upgrade. - (_major, _minor, _patch) => return None, - }; - - Some(start) -} - -/// Causes a compile-time error if the latest package cannot be upgraded. -/// -/// This serves as a reminder to consider the proper dumpless upgrade implementation when changing the package version. -const _CHECK_PACKAGE_CAN_UPGRADE: () = { - if start((VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)).is_none() { - panic!("cannot upgrade from latest package version") - } -}; - /// Return true if the cached stats of the index must be regenerated pub fn upgrade( wtxn: &mut RwTxn, @@ -113,79 +46,34 @@ pub fn upgrade( where MSP: Fn() -> bool + Sync, { - let from = index.get_version(wtxn)?.unwrap_or(db_version); + let upgrade_functions = UPGRADE_FUNCTIONS; - let start = - start(from).ok_or_else(|| InternalError::CannotUpgradeToVersion(from.0, from.1, from.2))?; + let initial_version = index.get_version(wtxn)?.unwrap_or(db_version); enum UpgradeVersion {} - let upgrade_path = &UPGRADE_FUNCTIONS[start..]; - let mut current_version = from; let mut regenerate_stats = false; - for (i, upgrade) in upgrade_path.iter().enumerate() { + for (i, upgrade) in upgrade_functions.iter().enumerate() { if (must_stop_processing)() { return Err(crate::Error::InternalError(InternalError::AbortedIndexation)); } - let target = upgrade.target_version(); - progress.update_progress(VariableNameStep::::new( - format!( - "Upgrading from v{}.{}.{} to v{}.{}.{}", - current_version.0, - current_version.1, - current_version.2, - target.0, - target.1, - target.2 - ), - i as u32, - upgrade_path.len() as u32, - )); - regenerate_stats |= upgrade.upgrade(wtxn, index, from, progress.clone())?; - index.put_version(wtxn, target)?; - current_version = target; + if upgrade.must_upgrade(initial_version) { + regenerate_stats |= upgrade.upgrade(wtxn, index, progress.clone())?; + progress.update_progress(VariableNameStep::::new( + upgrade.description(), + i as u32, + upgrade_functions.len() as u32, + )); + } else { + progress.update_progress(VariableNameStep::::new( + "Skipping migration that must not be applied", + i as u32, + upgrade_functions.len() as u32, + )); + } } + index.put_version(wtxn, (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH))?; + Ok(regenerate_stats) } - -#[allow(non_camel_case_types)] -struct ToCurrentNoOp {} - -impl UpgradeIndex for ToCurrentNoOp { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) - } -} - -/// Perform no operation during the upgrade except changing to the specified target version. -#[allow(non_camel_case_types)] -struct ToTargetNoOp { - pub target: (u32, u32, u32), -} - -impl UpgradeIndex for ToTargetNoOp { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) - } - - fn target_version(&self) -> (u32, u32, u32) { - self.target - } -} diff --git a/crates/milli/src/update/upgrade/v1_12.rs b/crates/milli/src/update/upgrade/v1_12.rs index f46e7f745..45247a738 100644 --- a/crates/milli/src/update/upgrade/v1_12.rs +++ b/crates/milli/src/update/upgrade/v1_12.rs @@ -4,17 +4,10 @@ use super::UpgradeIndex; use crate::progress::Progress; use crate::{make_enum_progress, Index, Result}; -#[allow(non_camel_case_types)] -pub(super) struct V1_12_To_V1_12_3 {} +pub(super) struct FixFieldDistribution {} -impl UpgradeIndex for V1_12_To_V1_12_3 { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _original: (u32, u32, u32), - progress: Progress, - ) -> Result { +impl UpgradeIndex for FixFieldDistribution { + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result { make_enum_progress! { enum FieldDistribution { RebuildingFieldDistribution, @@ -25,27 +18,28 @@ impl UpgradeIndex for V1_12_To_V1_12_3 { Ok(true) } - fn target_version(&self) -> (u32, u32, u32) { - (1, 12, 3) + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 12, 3) + } + + fn description(&self) -> &'static str { + "Recomputing field distribution which was wrong before v1.12.3" } } -#[allow(non_camel_case_types)] -pub(super) struct V1_12_3_To_V1_13_0 {} +pub(super) struct RecomputeStats {} -impl UpgradeIndex for V1_12_3_To_V1_13_0 { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { +impl UpgradeIndex for RecomputeStats { + fn upgrade(&self, _wtxn: &mut RwTxn, _index: &Index, _progress: Progress) -> Result { // recompute the indexes stats Ok(true) } - fn target_version(&self) -> (u32, u32, u32) { - (1, 13, 0) + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 13, 0) + } + + fn description(&self) -> &'static str { + "Recomputing stats" } } diff --git a/crates/milli/src/update/upgrade/v1_13.rs b/crates/milli/src/update/upgrade/v1_13.rs index 7f6608970..2ce60ccc7 100644 --- a/crates/milli/src/update/upgrade/v1_13.rs +++ b/crates/milli/src/update/upgrade/v1_13.rs @@ -5,17 +5,10 @@ use crate::database_stats::DatabaseStats; use crate::progress::Progress; use crate::{make_enum_progress, Index, Result}; -#[allow(non_camel_case_types)] -pub(super) struct V1_13_0_To_V1_13_1(); +pub(super) struct AddNewStats(); -impl UpgradeIndex for V1_13_0_To_V1_13_1 { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _original: (u32, u32, u32), - progress: Progress, - ) -> Result { +impl UpgradeIndex for AddNewStats { + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result { make_enum_progress! { enum DocumentsStats { CreatingDocumentsStats, @@ -30,26 +23,11 @@ impl UpgradeIndex for V1_13_0_To_V1_13_1 { Ok(true) } - fn target_version(&self) -> (u32, u32, u32) { - (1, 13, 1) - } -} - -#[allow(non_camel_case_types)] -pub(super) struct V1_13_1_To_Latest_V1_13(); - -impl UpgradeIndex for V1_13_1_To_Latest_V1_13 { - fn upgrade( - &self, - _wtxn: &mut RwTxn, - _index: &Index, - _original: (u32, u32, u32), - _progress: Progress, - ) -> Result { - Ok(false) + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 13, 1) } - fn target_version(&self) -> (u32, u32, u32) { - (1, 13, 3) + fn description(&self) -> &'static str { + "Computing newly introduced document stats" } -} +} \ No newline at end of file diff --git a/crates/milli/src/update/upgrade/v1_14.rs b/crates/milli/src/update/upgrade/v1_14.rs index 9950be706..55b9cc62b 100644 --- a/crates/milli/src/update/upgrade/v1_14.rs +++ b/crates/milli/src/update/upgrade/v1_14.rs @@ -5,17 +5,10 @@ use super::UpgradeIndex; use crate::progress::Progress; use crate::{make_enum_progress, Index, Result}; -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_13_To_Latest_V1_14(); +pub(super) struct UpgradeArroyVersion(); -impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _original: (u32, u32, u32), - progress: Progress, - ) -> Result { +impl UpgradeIndex for UpgradeArroyVersion { + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result { make_enum_progress! { enum VectorStore { UpdateInternalVersions, @@ -35,7 +28,11 @@ impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 { Ok(false) } - fn target_version(&self) -> (u32, u32, u32) { - (1, 14, 0) + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 14, 0) + } + + fn description(&self) -> &'static str { + "Updating vector store with an internal version" } } diff --git a/crates/milli/src/update/upgrade/v1_15.rs b/crates/milli/src/update/upgrade/v1_15.rs index 3457e69ba..ea31c8a72 100644 --- a/crates/milli/src/update/upgrade/v1_15.rs +++ b/crates/milli/src/update/upgrade/v1_15.rs @@ -7,25 +7,21 @@ use crate::progress::Progress; use crate::update::new::indexer::recompute_word_fst_from_word_docids_database; use crate::{Index, Result}; -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_14_To_Latest_V1_15(); +pub(super) struct RecomputeWordFst(); -impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _original: (u32, u32, u32), - progress: Progress, - ) -> Result { +impl UpgradeIndex for RecomputeWordFst { + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result { // Recompute the word FST from the word docids database. recompute_word_fst_from_word_docids_database(index, wtxn, &progress)?; Ok(false) } + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 15, 0) + } - fn target_version(&self) -> (u32, u32, u32) { - (1, 15, 0) + fn description(&self) -> &'static str { + "Recomputing word FST from word docids database as it was wrong before v1.15.0" } } diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs index f43efd77d..2bf67b5bd 100644 --- a/crates/milli/src/update/upgrade/v1_16.rs +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -6,15 +6,13 @@ use crate::progress::Progress; use crate::vector::db::{EmbedderInfo, EmbeddingStatus}; use crate::{Index, InternalError, Result}; -#[allow(non_camel_case_types)] -pub(super) struct Latest_V1_15_To_V1_16_0(); +pub(super) struct SwitchToMultimodal(); -impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { +impl UpgradeIndex for SwitchToMultimodal { fn upgrade( &self, wtxn: &mut RwTxn, index: &Index, - _original: (u32, u32, u32), _progress: Progress, ) -> Result { let v1_15_indexing_configs = index @@ -41,8 +39,11 @@ impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { Ok(false) } + fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool { + initial_version < (1, 16, 0) + } - fn target_version(&self) -> (u32, u32, u32) { - (1, 16, 0) + fn description(&self) -> &'static str { + "Migrating the database for multimodal support" } } From 4833da9edb37837750a76028a5cb04f23853c172 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Dec 2025 22:48:36 +0100 Subject: [PATCH 4/5] Chore: remove some duplicated lambdas to ease compile time --- crates/benchmarks/benches/indexing.rs | 128 +++++++++--------- .../milli/src/update/index_documents/mod.rs | 58 ++++---- 2 files changed, 97 insertions(+), 89 deletions(-) diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 75805e28c..56b8d298b 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -21,6 +21,10 @@ use roaring::RoaringBitmap; #[global_allocator] static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; +fn no_cancel() -> bool { + false +} + const BENCHMARK_ITERATION: usize = 10; fn setup_dir(path: impl AsRef) { @@ -65,7 +69,7 @@ fn setup_settings<'t>( let sortable_fields = sortable_fields.iter().map(|s| s.to_string()).collect(); builder.set_sortable_fields(sortable_fields); - builder.execute(&|| false, &Progress::default(), Default::default()).unwrap(); + builder.execute(&no_cancel, &Progress::default(), Default::default()).unwrap(); } fn setup_index_with_settings( @@ -152,7 +156,7 @@ fn indexing_songs_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -168,7 +172,7 @@ fn indexing_songs_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -220,7 +224,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -236,7 +240,7 @@ fn reindexing_songs_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -266,7 +270,7 @@ fn reindexing_songs_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -282,7 +286,7 @@ fn reindexing_songs_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -336,7 +340,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -352,7 +356,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -414,7 +418,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -430,7 +434,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -460,7 +464,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -476,7 +480,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -502,7 +506,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -518,7 +522,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -571,7 +575,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -587,7 +591,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -639,7 +643,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -655,7 +659,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -707,7 +711,7 @@ fn indexing_wiki(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -723,7 +727,7 @@ fn indexing_wiki(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -774,7 +778,7 @@ fn reindexing_wiki(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -790,7 +794,7 @@ fn reindexing_wiki(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -820,7 +824,7 @@ fn reindexing_wiki(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -836,7 +840,7 @@ fn reindexing_wiki(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -889,7 +893,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -905,7 +909,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -967,7 +971,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -983,7 +987,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1014,7 +1018,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1030,7 +1034,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1057,7 +1061,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1073,7 +1077,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1125,7 +1129,7 @@ fn indexing_movies_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1141,7 +1145,7 @@ fn indexing_movies_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1192,7 +1196,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1208,7 +1212,7 @@ fn reindexing_movies_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1238,7 +1242,7 @@ fn reindexing_movies_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1254,7 +1258,7 @@ fn reindexing_movies_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1307,7 +1311,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -1323,7 +1327,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -1372,7 +1376,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec bool { + false + } + #[test] fn simple_document_replacement() { let index = TempIndex::new(); @@ -1985,7 +1989,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2038,7 +2042,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2057,7 +2061,7 @@ mod tests { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2127,7 +2131,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2146,7 +2150,7 @@ mod tests { primary_key, &document_changes, RuntimeEmbedders::default(), - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2317,7 +2321,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2333,7 +2337,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2381,7 +2385,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2397,7 +2401,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2436,7 +2440,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2452,7 +2456,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2490,7 +2494,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2506,7 +2510,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2546,7 +2550,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2562,7 +2566,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2607,7 +2611,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2623,7 +2627,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2661,7 +2665,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2677,7 +2681,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2715,7 +2719,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2731,7 +2735,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2927,7 +2931,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -2943,7 +2947,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -2988,7 +2992,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -3004,7 +3008,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) @@ -3046,7 +3050,7 @@ mod tests { &rtxn, None, &mut new_fields_ids_map, - &|| false, + &no_cancel, Progress::default(), None, ) @@ -3062,7 +3066,7 @@ mod tests { primary_key, &document_changes, embedders, - &|| false, + &no_cancel, &Progress::default(), &Default::default(), ) From 8fa253c293e5365ea0c866f59a6c76ce77f4cf4c Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 3 Dec 2025 23:00:32 +0100 Subject: [PATCH 5/5] fmt --- crates/milli/src/update/upgrade/v1_13.rs | 2 +- crates/milli/src/update/upgrade/v1_16.rs | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/milli/src/update/upgrade/v1_13.rs b/crates/milli/src/update/upgrade/v1_13.rs index 2ce60ccc7..1b06a6930 100644 --- a/crates/milli/src/update/upgrade/v1_13.rs +++ b/crates/milli/src/update/upgrade/v1_13.rs @@ -30,4 +30,4 @@ impl UpgradeIndex for AddNewStats { fn description(&self) -> &'static str { "Computing newly introduced document stats" } -} \ No newline at end of file +} diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs index 2bf67b5bd..bec2320c2 100644 --- a/crates/milli/src/update/upgrade/v1_16.rs +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -9,12 +9,7 @@ use crate::{Index, InternalError, Result}; pub(super) struct SwitchToMultimodal(); impl UpgradeIndex for SwitchToMultimodal { - fn upgrade( - &self, - wtxn: &mut RwTxn, - index: &Index, - _progress: Progress, - ) -> Result { + fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, _progress: Progress) -> Result { let v1_15_indexing_configs = index .main .remap_types::>>()