diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index ad1054b8a..ae6532ef9 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -104,3 +104,20 @@ jobs: repository: meilisearch/meilisearch-cloud event-type: cloud-docker-build client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }' + + # Send notification to Swarmia to notify of a deployment: https://app.swarmia.com + - name: Send deployment to Swarmia + if: github.event_name == 'push' && success() + run: | + JSON_STRING=$( jq --null-input --compact-output \ + --arg version "${{ github.ref_name }}" \ + --arg appName "meilisearch" \ + --arg environment "production" \ + --arg commitSha "${{ github.sha }}" \ + --arg repositoryFullName "${{ github.repository }}" \ + '{"version": $version, "appName": $appName, "environment": $environment, "commitSha": $commitSha, "repositoryFullName": $repositoryFullName}' ) + + curl -H "Authorization: ${{ secrets.SWARMIA_DEPLOYMENTS_AUTHORIZATION }}" \ + -H "Content-Type: application/json" \ + -d "$JSON_STRING" \ + https://hook.swarmia.com/deployments diff --git a/Cargo.lock b/Cargo.lock index 6be777329..5b23f7e83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "bumpalo", @@ -677,7 +677,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "time", @@ -1652,7 +1652,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "big_s", @@ -1854,7 +1854,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.14.0" +version = "1.15.0" dependencies = [ "tempfile", "thiserror 2.0.9", @@ -1876,7 +1876,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.14.0" +version = "1.15.0" dependencies = [ "insta", "nom", @@ -1896,7 +1896,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.14.0" +version = "1.15.0" dependencies = [ "criterion", "serde_json", @@ -2035,7 +2035,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.14.0" +version = "1.15.0" dependencies = [ "arbitrary", "bumpalo", @@ -2738,7 +2738,7 @@ checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" [[package]] name = "index-scheduler" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "big_s", @@ -2947,7 +2947,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.14.0" +version = "1.15.0" dependencies = [ "criterion", "serde_json", @@ -3586,7 +3586,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.14.0" +version = "1.15.0" dependencies = [ "insta", "md5", @@ -3595,7 +3595,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.14.0" +version = "1.15.0" dependencies = [ "actix-cors", "actix-http", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.14.0" +version = "1.15.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3706,7 +3706,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.14.0" +version = "1.15.0" dependencies = [ "actix-web", "anyhow", @@ -3740,7 +3740,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "clap", @@ -3774,7 +3774,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.14.0" +version = "1.15.0" dependencies = [ "allocator-api2", "arroy", @@ -4287,7 +4287,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.14.0" +version = "1.15.0" dependencies = [ "big_s", "serde_json", @@ -6898,7 +6898,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.14.0" +version = "1.15.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index b023a778e..ce4b806f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ ] [workspace.package] -version = "1.14.0" +version = "1.15.0" authors = [ "Quentin de Quelen ", "Clément Renault ", diff --git a/README.md b/README.md index 508efb14b..77eecde25 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ - [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos). - [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination. - [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs. -- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application. +- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application. See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos). @@ -99,7 +99,7 @@ If you want to know more about the kind of data we collect and what we use it fo ## 📫 Get in touch! -Meilisearch is a search engine created by [Meili]([https://www.welcometothejungle.com/en/companies/meilisearch](https://www.meilisearch.com/careers)), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact) +Meilisearch is a search engine created by [Meili](https://www.meilisearch.com/careers), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact) 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months. diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 6b63e7c6b..b4a4fcb24 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -373,6 +373,7 @@ impl From> for v6::Settings { }, disable_on_words: typo.disable_on_words.into(), disable_on_attributes: typo.disable_on_attributes.into(), + disable_on_numbers: v6::Setting::NotSet, }), v5::Setting::Reset => v6::Setting::Reset, v5::Setting::NotSet => v6::Setting::NotSet, diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs index 280127d04..cb798b385 100644 --- a/crates/index-scheduler/src/error.rs +++ b/crates/index-scheduler/src/error.rs @@ -2,6 +2,7 @@ use std::fmt::Display; use meilisearch_types::batches::BatchId; use meilisearch_types::error::{Code, ErrorCode}; +use meilisearch_types::milli::index::RollbackOutcome; use meilisearch_types::tasks::{Kind, Status}; use meilisearch_types::{heed, milli}; use thiserror::Error; @@ -150,8 +151,24 @@ pub enum Error { CorruptedTaskQueue, #[error(transparent)] DatabaseUpgrade(Box), + #[error("Failed to rollback for index `{index}`: {rollback_outcome} ")] + RollbackFailed { index: String, rollback_outcome: RollbackOutcome }, #[error(transparent)] UnrecoverableError(Box), + #[error("The index scheduler is in version v{}.{}.{}, but Meilisearch is in version v{}.{}.{}.\n - hint: start the correct version of Meilisearch, or consider updating your database. See also ", + index_scheduler_version.0, index_scheduler_version.1, index_scheduler_version.2, + package_version.0, package_version.1, package_version.2)] + IndexSchedulerVersionMismatch { + index_scheduler_version: (u32, u32, u32), + package_version: (u32, u32, u32), + }, + #[error("Index `{index}` is in version v{}.{}.{}, but Meilisearch is in version v{}.{}.{}.\n - note: this is an internal error, please consider filing a bug report: ", + index_version.0, index_version.1, index_version.2, package_version.0, package_version.1, package_version.2)] + IndexVersionMismatch { + index: String, + index_version: (u32, u32, u32), + package_version: (u32, u32, u32), + }, #[error(transparent)] HeedTransaction(heed::Error), @@ -209,6 +226,9 @@ impl Error { | Error::CorruptedTaskQueue | Error::DatabaseUpgrade(_) | Error::UnrecoverableError(_) + | Error::IndexSchedulerVersionMismatch { .. } + | Error::IndexVersionMismatch { .. } + | Error::RollbackFailed { .. } | Error::HeedTransaction(_) => false, #[cfg(test)] Error::PlannedFailure => false, @@ -274,7 +294,10 @@ impl ErrorCode for Error { Error::CorruptedTaskQueue => Code::Internal, Error::CorruptedDump => Code::Internal, Error::DatabaseUpgrade(_) => Code::Internal, + Error::RollbackFailed { .. } => Code::Internal, Error::UnrecoverableError(_) => Code::Internal, + Error::IndexSchedulerVersionMismatch { .. } => Code::Internal, + Error::IndexVersionMismatch { .. } => Code::Internal, Error::CreateBatch(_) => Code::Internal, // This one should never be seen by the end user diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index c1f6ff472..86fb17ca7 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -7,6 +7,7 @@ use meilisearch_types::heed::types::{SerdeJson, Str}; use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls}; use meilisearch_types::milli; use meilisearch_types::milli::database_stats::DatabaseStats; +use meilisearch_types::milli::index::RollbackOutcome; use meilisearch_types::milli::update::IndexerConfig; use meilisearch_types::milli::{FieldDistribution, Index}; use serde::{Deserialize, Serialize}; @@ -431,6 +432,51 @@ impl IndexMapper { Ok(index) } + pub fn rollback_index( + &self, + rtxn: &RoTxn, + name: &str, + to: (u32, u32, u32), + ) -> Result { + // remove any currently updating index to make sure that we aren't keeping a reference to the index somewhere + drop(self.currently_updating_index.write().unwrap().take()); + + let uuid = self + .index_mapping + .get(rtxn, name)? + .ok_or_else(|| Error::IndexNotFound(name.to_string()))?; + + // take the lock to make sure noone is messing with the indexes while we rollback + // this will block any search or other operation, but we are rollbacking so this is probably acceptable. + let mut index_map = self.index_map.write().unwrap(); + + 'close_index: loop { + match index_map.get(&uuid) { + Available(_) => { + index_map.close_for_resize(&uuid, self.enable_mdb_writemap, 0); + // index should now be `Closing`; try again + continue; + } + // index already closed + Missing => break 'close_index, + // closing requested by this thread or another one; wait for closing to complete, then exit + Closing(closing_index) => { + if closing_index.wait_timeout(Duration::from_secs(100)).is_none() { + // release the lock so it doesn't get poisoned + drop(index_map); + panic!("cannot close index") + } + break; + } + BeingDeleted => return Err(Error::IndexNotFound(name.to_string())), + }; + } + + let index_path = self.base_path.join(uuid.to_string()); + Index::rollback(milli::heed::EnvOpenOptions::new().read_txn_without_tls(), index_path, to) + .map_err(|err| crate::Error::from_milli(err, Some(name.to_string()))) + } + /// Attempts `f` for each index that exists in the index mapper. /// /// It is preferable to use this function rather than a loop that opens all indexes, as a way to avoid having all indexes opened, diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index ee271b5df..89e615132 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -41,11 +41,8 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { let mut snap = String::new(); let indx_sched_version = version.get_version(&rtxn).unwrap(); - let latest_version = ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), - ); + let latest_version = + (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH); if indx_sched_version != Some(latest_version) { snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n")); } diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index a5afdcbf9..4f1109348 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -398,9 +398,9 @@ impl IndexScheduler { Ok(Ok(TickOutcome::StopProcessingForever)) => break, Ok(Err(e)) => { tracing::error!("{e}"); - // Wait one second when an irrecoverable error occurs. + // Wait when an irrecoverable error occurs. if !e.is_recoverable() { - std::thread::sleep(Duration::from_secs(1)); + std::thread::sleep(Duration::from_secs(10)); } } Err(_panic) => { diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index 09ce46884..f23b811e5 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -74,6 +74,7 @@ make_enum_progress! { make_enum_progress! { pub enum TaskCancelationProgress { RetrievingTasks, + CancelingUpgrade, UpdatingTasks, } } diff --git a/crates/index-scheduler/src/scheduler/create_batch.rs b/crates/index-scheduler/src/scheduler/create_batch.rs index 29d352fe8..e3763881b 100644 --- a/crates/index-scheduler/src/scheduler/create_batch.rs +++ b/crates/index-scheduler/src/scheduler/create_batch.rs @@ -423,7 +423,8 @@ impl IndexScheduler { } /// Create the next batch to be processed; - /// 1. We get the *last* task to cancel. + /// 0. We get the *last* task to cancel. + /// 1. We get the tasks to upgrade. /// 2. We get the *next* task to delete. /// 3. We get the *next* snapshot to process. /// 4. We get the *next* dump to process. @@ -443,7 +444,20 @@ impl IndexScheduler { let count_total_enqueued = enqueued.len(); let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?; - // 0. The priority over everything is to upgrade the instance + // 0. we get the last task to cancel. + let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued; + if let Some(task_id) = to_cancel.max() { + let mut task = + self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?; + current_batch.processing(Some(&mut task)); + current_batch.reason(BatchStopReason::TaskCannotBeBatched { + kind: Kind::TaskCancelation, + id: task_id, + }); + return Ok(Some((Batch::TaskCancelation { task }, current_batch))); + } + + // 1. We upgrade the instance // There shouldn't be multiple upgrade tasks but just in case we're going to batch all of them at the same time let upgrade = self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)? & (enqueued | failed); if !upgrade.is_empty() { @@ -459,17 +473,21 @@ impl IndexScheduler { return Ok(Some((Batch::UpgradeDatabase { tasks }, current_batch))); } - // 1. we get the last task to cancel. - let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued; - if let Some(task_id) = to_cancel.max() { - let mut task = - self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?; - current_batch.processing(Some(&mut task)); - current_batch.reason(BatchStopReason::TaskCannotBeBatched { - kind: Kind::TaskCancelation, - id: task_id, - }); - return Ok(Some((Batch::TaskCancelation { task }, current_batch))); + // check the version of the scheduler here. + // if the version is not the current, refuse to batch any additional task. + let version = self.version.get_version(rtxn)?; + let package_version = ( + meilisearch_types::versioning::VERSION_MAJOR, + meilisearch_types::versioning::VERSION_MINOR, + meilisearch_types::versioning::VERSION_PATCH, + ); + if version != Some(package_version) { + return Err(Error::UnrecoverableError(Box::new( + Error::IndexSchedulerVersionMismatch { + index_scheduler_version: version.unwrap_or((1, 12, 0)), + package_version, + }, + ))); } // 2. we get the next task to delete diff --git a/crates/index-scheduler/src/scheduler/process_batch.rs b/crates/index-scheduler/src/scheduler/process_batch.rs index 42de1d137..c349f90ad 100644 --- a/crates/index-scheduler/src/scheduler/process_batch.rs +++ b/crates/index-scheduler/src/scheduler/process_batch.rs @@ -6,7 +6,8 @@ use meilisearch_types::batches::{BatchEnqueuedAt, BatchId}; use meilisearch_types::heed::{RoTxn, RwTxn}; use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::{self, ChannelCongestion}; -use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task}; +use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task}; +use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use milli::update::Settings as MilliSettings; use roaring::RoaringBitmap; @@ -144,11 +145,22 @@ impl IndexScheduler { self.index_mapper.index(&rtxn, &index_uid)? }; + let mut index_wtxn = index.write_txn()?; + + let index_version = index.get_version(&index_wtxn)?.unwrap_or((1, 12, 0)); + let package_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); + if index_version != package_version { + return Err(Error::IndexVersionMismatch { + index: index_uid, + index_version, + package_version, + }); + } + // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick self.index_mapper .set_currently_updating_index(Some((index_uid.clone(), index.clone()))); - let mut index_wtxn = index.write_txn()?; let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?; let (tasks, congestion) = self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?; @@ -353,9 +365,11 @@ impl IndexScheduler { let KindWithContent::UpgradeDatabase { from } = tasks.last().unwrap().kind else { unreachable!(); }; + let ret = catch_unwind(AssertUnwindSafe(|| self.process_upgrade(from, progress))); match ret { Ok(Ok(())) => (), + Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask), Ok(Err(e)) => return Err(Error::DatabaseUpgrade(Box::new(e))), Err(e) => { let msg = match e.downcast_ref::<&'static str>() { @@ -653,17 +667,79 @@ impl IndexScheduler { progress: &Progress, ) -> Result> { progress.update_progress(TaskCancelationProgress::RetrievingTasks); + let mut tasks_to_cancel = RoaringBitmap::new(); + let enqueued_tasks = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?; + + // 0. Check if any upgrade task was matched. + // If so, we cancel all the failed or enqueued upgrade tasks. + let upgrade_tasks = &self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)?; + let is_canceling_upgrade = !matched_tasks.is_disjoint(upgrade_tasks); + if is_canceling_upgrade { + let failed_tasks = self.queue.tasks.get_status(rtxn, Status::Failed)?; + tasks_to_cancel |= upgrade_tasks & (enqueued_tasks | failed_tasks); + } // 1. Remove from this list the tasks that we are not allowed to cancel // Notice that only the _enqueued_ ones are cancelable and we should // have already aborted the indexation of the _processing_ ones - let cancelable_tasks = self.queue.tasks.get_status(rtxn, Status::Enqueued)?; - let tasks_to_cancel = cancelable_tasks & matched_tasks; + tasks_to_cancel |= enqueued_tasks & matched_tasks; + // 2. If we're canceling an upgrade, attempt the rollback + if let Some(latest_upgrade_task) = (&tasks_to_cancel & upgrade_tasks).max() { + progress.update_progress(TaskCancelationProgress::CancelingUpgrade); + + let task = self.queue.tasks.get_task(rtxn, latest_upgrade_task)?.unwrap(); + let Some(Details::UpgradeDatabase { from, to }) = task.details else { + unreachable!("wrong details for upgrade task {latest_upgrade_task}") + }; + + // check that we are rollbacking an upgrade to the current Meilisearch + let bin_major: u32 = meilisearch_types::versioning::VERSION_MAJOR; + let bin_minor: u32 = meilisearch_types::versioning::VERSION_MINOR; + let bin_patch: u32 = meilisearch_types::versioning::VERSION_PATCH; + + if to == (bin_major, bin_minor, bin_patch) { + tracing::warn!( + "Rollbacking from v{}.{}.{} to v{}.{}.{}", + to.0, + to.1, + to.2, + from.0, + from.1, + from.2 + ); + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.process_rollback(from, progress) + })) { + Ok(Ok(())) => {} + Ok(Err(err)) => return Err(Error::DatabaseUpgrade(Box::new(err))), + Err(e) => { + let msg = match e.downcast_ref::<&'static str>() { + Some(s) => *s, + None => match e.downcast_ref::() { + Some(s) => &s[..], + None => "Box", + }, + }; + return Err(Error::DatabaseUpgrade(Box::new(Error::ProcessBatchPanicked( + msg.to_string(), + )))); + } + } + } else { + tracing::debug!( + "Not rollbacking an upgrade targetting the earlier version v{}.{}.{}", + bin_major, + bin_minor, + bin_patch + ) + } + } + + // 3. We now have a list of tasks to cancel, cancel them let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32); progress.update_progress(progress_obj); - // 2. We now have a list of tasks to cancel, cancel them let mut tasks = self.queue.tasks.get_existing_tasks( rtxn, tasks_to_cancel.iter().inspect(|_| { diff --git a/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs index 4feebabc4..6fbdb0b58 100644 --- a/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs +++ b/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs @@ -12,10 +12,14 @@ impl IndexScheduler { #[cfg(test)] self.maybe_fail(crate::test_utils::FailureLocation::ProcessUpgrade)?; - enum UpgradeIndex {} let indexes = self.index_names()?; for (i, uid) in indexes.iter().enumerate() { + let must_stop_processing = self.scheduler.must_stop_processing.clone(); + + if must_stop_processing.get() { + return Err(Error::AbortedTask); + } progress.update_progress(VariableNameStep::::new( format!("Upgrading index `{uid}`"), i as u32, @@ -27,6 +31,7 @@ impl IndexScheduler { &mut index_wtxn, &index, db_version, + || must_stop_processing.get(), progress.clone(), ) .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; @@ -46,4 +51,42 @@ impl IndexScheduler { Ok(()) } + + pub fn process_rollback(&self, db_version: (u32, u32, u32), progress: &Progress) -> Result<()> { + let mut wtxn = self.env.write_txn()?; + tracing::info!(?db_version, "roll back index scheduler version"); + self.version.set_version(&mut wtxn, db_version)?; + let db_path = self.scheduler.version_file_path.parent().unwrap(); + wtxn.commit()?; + + let indexes = self.index_names()?; + + tracing::info!("roll backing all indexes"); + for (i, uid) in indexes.iter().enumerate() { + progress.update_progress(VariableNameStep::::new( + format!("Rollbacking index `{uid}`"), + i as u32, + indexes.len() as u32, + )); + let index_schd_rtxn = self.env.read_txn()?; + + let rollback_outcome = + self.index_mapper.rollback_index(&index_schd_rtxn, uid, db_version)?; + if !rollback_outcome.succeeded() { + return Err(crate::Error::RollbackFailed { index: uid.clone(), rollback_outcome }); + } + } + + tracing::info!(?db_path, ?db_version, "roll back version file"); + meilisearch_types::versioning::create_version_file( + db_path, + db_version.0, + db_version.1, + db_version.2, + )?; + + Ok(()) + } } + +enum UpgradeIndex {} diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 552a4bc0e..178ec8166 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "task with id 1 of type `indexCreation` cannot be batched", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 2 of type `indexCreation` cannot be batched", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 3 of type `indexCreation` cannot be batched", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index 6d6276067..37bb9d78e 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 7ef9c42e1..fd8656c42 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 24b498ed2..899a507f5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 012e4b215..e3244fc28 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 813e4255b..9d78f6bbf 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index 5c04a66ff..0d44b3c81 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -114,12 +114,8 @@ impl IndexScheduler { auto_upgrade: true, // Don't cost much and will ensure the happy path works embedding_cache_cap: 10, }; - let version = configuration(&mut options).unwrap_or_else(|| { - ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), - ) + let version = configuration(&mut options).unwrap_or({ + (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH) }); std::fs::create_dir_all(&options.auth_path).unwrap(); diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index 4a3cb2f75..2053caa92 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -28,12 +28,17 @@ pub fn upgrade_index_scheduler( let current_minor = to.1; let current_patch = to.2; - let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[&ToCurrentNoOp {}]; + let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[ + // This is the last upgrade function, it will be called when the index is up to date. + // any other upgrade function should be added before this one. + &ToCurrentNoOp {}, + ]; let start = match from { (1, 12, _) => 0, (1, 13, _) => 0, (1, 14, _) => 0, + (1, 15, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) @@ -104,10 +109,6 @@ impl UpgradeIndexScheduler for ToCurrentNoOp { } fn target_version(&self) -> (u32, u32, u32) { - ( - VERSION_MAJOR.parse().unwrap(), - VERSION_MINOR.parse().unwrap(), - VERSION_PATCH.parse().unwrap(), - ) + (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) } } diff --git a/crates/index-scheduler/src/versioning.rs b/crates/index-scheduler/src/versioning.rs index 107b8e0ba..b0cb7fdb5 100644 --- a/crates/index-scheduler/src/versioning.rs +++ b/crates/index-scheduler/src/versioning.rs @@ -39,9 +39,9 @@ impl Versioning { } pub fn set_current_version(&self, wtxn: &mut RwTxn) -> Result<(), heed::Error> { - let major = versioning::VERSION_MAJOR.parse().unwrap(); - let minor = versioning::VERSION_MINOR.parse().unwrap(); - let patch = versioning::VERSION_PATCH.parse().unwrap(); + let major = versioning::VERSION_MAJOR; + let minor = versioning::VERSION_MINOR; + let patch = versioning::VERSION_PATCH; self.set_version(wtxn, (major, minor, patch)) } @@ -64,9 +64,9 @@ impl Versioning { }; wtxn.commit()?; - let bin_major: u32 = versioning::VERSION_MAJOR.parse().unwrap(); - let bin_minor: u32 = versioning::VERSION_MINOR.parse().unwrap(); - let bin_patch: u32 = versioning::VERSION_PATCH.parse().unwrap(); + let bin_major: u32 = versioning::VERSION_MAJOR; + let bin_minor: u32 = versioning::VERSION_MINOR; + let bin_patch: u32 = versioning::VERSION_PATCH; let to = (bin_major, bin_minor, bin_patch); if from != to { diff --git a/crates/meilisearch-types/src/batch_view.rs b/crates/meilisearch-types/src/batch_view.rs index 0767cdebf..791e1d4ec 100644 --- a/crates/meilisearch-types/src/batch_view.rs +++ b/crates/meilisearch-types/src/batch_view.rs @@ -22,7 +22,7 @@ pub struct BatchView { #[serde(with = "time::serde::rfc3339::option", default)] pub finished_at: Option, #[serde(default = "meilisearch_types::batches::default_stop_reason")] - pub batcher_stopped_because: String, + pub batch_creation_complete: String, } impl BatchView { @@ -35,7 +35,7 @@ impl BatchView { duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at), started_at: batch.started_at, finished_at: batch.finished_at, - batcher_stopped_because: batch.stop_reason.clone(), + batch_creation_complete: batch.stop_reason.clone(), } } } diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 6ace0f4ee..ccf0d75ee 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -8,6 +8,7 @@ use std::str::FromStr; use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef}; use fst::IntoStreamer; +use milli::disabled_typos_terms::DisabledTyposTerms; use milli::index::{IndexEmbeddingConfig, PrefixSearch}; use milli::proximity::ProximityPrecision; use milli::update::Setting; @@ -104,6 +105,10 @@ pub struct TypoSettings { #[deserr(default)] #[schema(value_type = Option>, example = json!(["uuid", "url"]))] pub disable_on_attributes: Setting>, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option, example = json!(true))] + pub disable_on_numbers: Setting, } #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] @@ -701,6 +706,12 @@ pub fn apply_settings_to_builder( Setting::Reset => builder.reset_exact_attributes(), Setting::NotSet => (), } + + match value.disable_on_numbers { + Setting::Set(val) => builder.set_disable_on_numbers(val), + Setting::Reset => builder.reset_disable_on_numbers(), + Setting::NotSet => (), + } } Setting::Reset => { // all typo settings need to be reset here. @@ -826,12 +837,14 @@ pub fn settings( }; let disabled_attributes = index.exact_attributes(rtxn)?.into_iter().map(String::from).collect(); + let DisabledTyposTerms { disable_on_numbers } = index.disabled_typos_terms(rtxn)?; let typo_tolerance = TypoSettings { enabled: Setting::Set(index.authorize_typos(rtxn)?), min_word_size_for_typos: Setting::Set(min_typo_word_len), disable_on_words: Setting::Set(disabled_words), disable_on_attributes: Setting::Set(disabled_attributes), + disable_on_numbers: Setting::Set(disable_on_numbers), }; let faceting = FacetingSettings { diff --git a/crates/meilisearch-types/src/tasks.rs b/crates/meilisearch-types/src/tasks.rs index d96a45992..6e10f2606 100644 --- a/crates/meilisearch-types/src/tasks.rs +++ b/crates/meilisearch-types/src/tasks.rs @@ -272,9 +272,9 @@ impl KindWithContent { KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: (from.0, from.1, from.2), to: ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), + versioning::VERSION_MAJOR, + versioning::VERSION_MINOR, + versioning::VERSION_PATCH, ), }), } @@ -338,9 +338,9 @@ impl KindWithContent { KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: *from, to: ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), + versioning::VERSION_MAJOR, + versioning::VERSION_MINOR, + versioning::VERSION_PATCH, ), }), } @@ -386,9 +386,9 @@ impl From<&KindWithContent> for Option
{ KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { from: *from, to: ( - versioning::VERSION_MAJOR.parse().unwrap(), - versioning::VERSION_MINOR.parse().unwrap(), - versioning::VERSION_PATCH.parse().unwrap(), + versioning::VERSION_MAJOR, + versioning::VERSION_MINOR, + versioning::VERSION_PATCH, ), }), } diff --git a/crates/meilisearch-types/src/versioning.rs b/crates/meilisearch-types/src/versioning.rs index 07e42c2ce..b2124c04e 100644 --- a/crates/meilisearch-types/src/versioning.rs +++ b/crates/meilisearch-types/src/versioning.rs @@ -8,9 +8,7 @@ use tempfile::NamedTempFile; /// The name of the file that contains the version of the database. pub const VERSION_FILE_NAME: &str = "VERSION"; -pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); -pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); -pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); +pub use milli::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; /// Persists the version of the current Meilisearch binary to a VERSION file pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> { @@ -19,9 +17,9 @@ pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> { pub fn create_version_file( db_path: &Path, - major: &str, - minor: &str, - patch: &str, + major: u32, + minor: u32, + patch: u32, ) -> anyhow::Result<()> { let version_path = db_path.join(VERSION_FILE_NAME); // In order to persist the file later we must create it in the `data.ms` and not in `/tmp` diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 9364bc83d..57ef6d6f2 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -236,10 +236,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< auto_upgrade: opt.experimental_dumpless_upgrade, embedding_cache_cap: opt.experimental_embedding_cache_entries, }; - let bin_major: u32 = VERSION_MAJOR.parse().unwrap(); - let bin_minor: u32 = VERSION_MINOR.parse().unwrap(); - let bin_patch: u32 = VERSION_PATCH.parse().unwrap(); - let binary_version = (bin_major, bin_minor, bin_patch); + let binary_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); let empty_db = is_empty_db(&opt.db_path); let (index_scheduler, auth_controller) = if let Some(ref snapshot_path) = opt.import_snapshot { diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index f676e8c0e..e775d1ea4 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -310,7 +310,7 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -353,7 +353,7 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -398,7 +398,7 @@ async fn test_summarized_delete_documents_by_batch() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -440,7 +440,7 @@ async fn test_summarized_delete_documents_by_batch() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -488,7 +488,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -532,7 +532,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -576,7 +576,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -622,7 +622,7 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -664,7 +664,7 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -731,7 +731,7 @@ async fn test_summarized_settings_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -773,7 +773,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 0 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id 0 of type `indexCreation` cannot be batched" } "###); @@ -812,7 +812,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 1 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched" } "###); } @@ -964,7 +964,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 0 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id 0 of type `indexUpdate` cannot be batched" } "###); @@ -1003,7 +1003,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 1 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id 1 of type `indexUpdate` cannot be batched" } "###); @@ -1043,7 +1043,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 3 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id 3 of type `indexUpdate` cannot be batched" } "###); @@ -1082,7 +1082,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 4 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id 4 of type `indexUpdate` cannot be batched" } "###); } @@ -1134,7 +1134,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 0 of type `indexSwap` cannot be batched" + "batchCreationComplete": "task with id 0 of type `indexSwap` cannot be batched" } "###); @@ -1177,7 +1177,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 1 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched" } "###); } @@ -1224,7 +1224,7 @@ async fn test_summarized_batch_cancelation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 1 of type `taskCancelation` cannot be batched" + "batchCreationComplete": "task with id 1 of type `taskCancelation` cannot be batched" } "###); } @@ -1271,7 +1271,7 @@ async fn test_summarized_batch_deletion() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "a batch of tasks of type `taskDeletion` cannot be batched with any other type of task" + "batchCreationComplete": "a batch of tasks of type `taskDeletion` cannot be batched with any other type of task" } "###); } @@ -1313,7 +1313,7 @@ async fn test_summarized_dump_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "task with id 0 of type `dumpCreation` cannot be batched" + "batchCreationComplete": "task with id 0 of type `dumpCreation` cannot be batched" } "###); } diff --git a/crates/meilisearch/tests/dumps/mod.rs b/crates/meilisearch/tests/dumps/mod.rs index e5aa52dc6..3ba3c20eb 100644 --- a/crates/meilisearch/tests/dumps/mod.rs +++ b/crates/meilisearch/tests/dumps/mod.rs @@ -87,7 +87,8 @@ async fn import_dump_v1_movie_raw() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -260,7 +261,8 @@ async fn import_dump_v1_movie_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -432,7 +434,8 @@ async fn import_dump_v1_rubygems_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -590,7 +593,8 @@ async fn import_dump_v2_movie_raw() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -760,7 +764,8 @@ async fn import_dump_v2_movie_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -929,7 +934,8 @@ async fn import_dump_v2_rubygems_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1087,7 +1093,8 @@ async fn import_dump_v3_movie_raw() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1257,7 +1264,8 @@ async fn import_dump_v3_movie_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1426,7 +1434,8 @@ async fn import_dump_v3_rubygems_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1584,7 +1593,8 @@ async fn import_dump_v4_movie_raw() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1754,7 +1764,8 @@ async fn import_dump_v4_movie_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -1923,7 +1934,8 @@ async fn import_dump_v4_rubygems_with_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -2212,7 +2224,8 @@ async fn import_dump_v6_containing_experimental_features() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -2444,7 +2457,8 @@ async fn generate_and_import_dump_containing_vectors() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, diff --git a/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap index 26736ad76..2f3b0a7f9 100644 --- a/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap +++ b/crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap @@ -27,7 +27,7 @@ source: crates/meilisearch/tests/dumps/mod.rs "duration": "[date]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" }, { "uid": 1, @@ -51,7 +51,7 @@ source: crates/meilisearch/tests/dumps/mod.rs "duration": "PT0.144827890S", "startedAt": "2025-02-04T10:15:21.275640274Z", "finishedAt": "2025-02-04T10:15:21.420468164Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 0, @@ -72,7 +72,7 @@ source: crates/meilisearch/tests/dumps/mod.rs "duration": "PT0.032902186S", "startedAt": "2025-02-04T10:14:43.559526162Z", "finishedAt": "2025-02-04T10:14:43.592428348Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 3, diff --git a/crates/meilisearch/tests/search/mod.rs b/crates/meilisearch/tests/search/mod.rs index f6e79dbb9..6d98c0b2a 100644 --- a/crates/meilisearch/tests/search/mod.rs +++ b/crates/meilisearch/tests/search/mod.rs @@ -1976,3 +1976,93 @@ async fn change_facet_casing() { }) .await; } + +#[actix_rt::test] +async fn test_exact_typos_terms() { + let documents = json!([ + { + "id": 0, + "title": "The zeroth document 1298484", + }, + { + "id": 1, + "title": "The first document 234342", + "nested": { + "object": "field 22231", + "machin": "bidule 23443.32111", + }, + }, + { + "id": 2, + "title": "The second document 3398499", + "nested": [ + "array", + { + "object": "field 23245121,23223", + }, + { + "prout": "truc 123980612321", + "machin": "lol 12345645333447879", + }, + ], + }, + { + "id": 3, + "title": "The third document 12333", + "nested": "I lied 98878", + }, + ]); + + // Test prefix search + test_settings_documents_indexing_swapping_and_search( + &documents, + &json!({ + "searchableAttributes": ["title", "nested.object", "nested.machin"], + "typoTolerance": { + "enabled": true, + "disableOnNumbers": true + } + }), + &json!({"q": "12345"}), + |response, code| { + assert_eq!(code, 200, "{}", response); + snapshot!(json_string!(response["hits"]), @r###" + [ + { + "id": 2, + "title": "The second document 3398499", + "nested": [ + "array", + { + "object": "field 23245121,23223" + }, + { + "prout": "truc 123980612321", + "machin": "lol 12345645333447879" + } + ] + } + ] + "###); + }, + ) + .await; + + // Test typo search + test_settings_documents_indexing_swapping_and_search( + &documents, + &json!({ + "searchableAttributes": ["title", "nested.object", "nested.machin"], + "typoTolerance": { + "enabled": true, + "disableOnNumbers": true + } + }), + &json!({"q": "123457"}), + |response, code| { + assert_eq!(code, 200, "{}", response); + snapshot!(json_string!(response["hits"]), @r###"[]"###); + }, + ) + .await; +} diff --git a/crates/meilisearch/tests/settings/errors.rs b/crates/meilisearch/tests/settings/errors.rs index ed1e0298f..4220cdbf8 100644 --- a/crates/meilisearch/tests/settings/errors.rs +++ b/crates/meilisearch/tests/settings/errors.rs @@ -274,7 +274,7 @@ async fn settings_bad_typo_tolerance() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r###" { - "message": "Unknown field `typoTolerance`: expected one of `enabled`, `minWordSizeForTypos`, `disableOnWords`, `disableOnAttributes`", + "message": "Unknown field `typoTolerance`: expected one of `enabled`, `minWordSizeForTypos`, `disableOnWords`, `disableOnAttributes`, `disableOnNumbers`", "code": "invalid_settings_typo_tolerance", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_settings_typo_tolerance" diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index fbb97f999..5c0f89ed3 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -179,7 +179,7 @@ test_setting_routes!( { setting: typo_tolerance, update_verb: patch, - default_value: {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": []} + default_value: {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [], "disableOnNumbers": false} }, ); @@ -276,7 +276,7 @@ async fn secrets_are_hidden_in_settings() { let (response, code) = index.settings().await; meili_snap::snapshot!(code, @"200 OK"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r#" + meili_snap::snapshot!(meili_snap::json_string!(response), @r###" { "displayedAttributes": [ "*" @@ -308,7 +308,8 @@ async fn secrets_are_hidden_in_settings() { "twoTypos": 9 }, "disableOnWords": [], - "disableOnAttributes": [] + "disableOnAttributes": [], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 100, @@ -337,7 +338,7 @@ async fn secrets_are_hidden_in_settings() { "facetSearch": true, "prefixSearch": "indexingTime" } - "#); + "###); let (response, code) = server.get_task(settings_update_uid).await; meili_snap::snapshot!(code, @"200 OK"); diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index ae6bcff40..f1e45164e 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.14.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.15.0"); } #[actix_rt::test] @@ -54,11 +54,11 @@ async fn version_requires_downgrade() { std::fs::create_dir_all(&db_path).unwrap(); let major = meilisearch_types::versioning::VERSION_MAJOR; let minor = meilisearch_types::versioning::VERSION_MINOR; - let patch = meilisearch_types::versioning::VERSION_PATCH.parse::().unwrap() + 1; + let patch = meilisearch_types::versioning::VERSION_PATCH + 1; std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.14.1 is higher than the Meilisearch version 1.14.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.15.1 is higher than the Meilisearch version 1.15.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap index e836fa4b3..af7e82c8b 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap @@ -1,6 +1,5 @@ --- source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs -snapshot_kind: text --- { "displayedAttributes": [ @@ -49,7 +48,8 @@ snapshot_kind: text ], "disableOnAttributes": [ "surname" - ] + ], + "disableOnNumbers": false }, "faceting": { "maxValuesPerFacet": 99, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index a28fb8789..1d89e6838 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "stats": { "totalNbTasks": 1, @@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" + "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" }, { "uid": 23, @@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.004146631S", "startedAt": "2025-01-23T11:38:57.012591321Z", "finishedAt": "2025-01-23T11:38:57.016737952Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 22, @@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.102738497S", "startedAt": "2025-01-23T11:36:22.551906856Z", "finishedAt": "2025-01-23T11:36:22.654645353Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 21, @@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.005108474S", "startedAt": "2025-01-23T11:36:04.132670526Z", "finishedAt": "2025-01-23T11:36:04.137779Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 20, @@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.027954894S", "startedAt": "2025-01-23T11:35:53.631082795Z", "finishedAt": "2025-01-23T11:35:53.659037689Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 19, @@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.006903297S", "startedAt": "2025-01-20T11:50:52.874106134Z", "finishedAt": "2025-01-20T11:50:52.881009431Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 18, @@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000481257S", "startedAt": "2025-01-20T11:48:04.92820416Z", "finishedAt": "2025-01-20T11:48:04.928685417Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 17, @@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000407005S", "startedAt": "2025-01-20T11:47:53.509403957Z", "finishedAt": "2025-01-20T11:47:53.509810962Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 16, @@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000403716S", "startedAt": "2025-01-20T11:47:48.430653005Z", "finishedAt": "2025-01-20T11:47:48.431056721Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 15, @@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000417016S", "startedAt": "2025-01-20T11:47:42.429678617Z", "finishedAt": "2025-01-20T11:47:42.430095633Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 14, @@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT12.086284842S", "startedAt": "2025-01-20T11:47:03.092181576Z", "finishedAt": "2025-01-20T11:47:15.178466418Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 13, @@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.011506614S", "startedAt": "2025-01-16T17:18:43.29334923Z", "finishedAt": "2025-01-16T17:18:43.304855844Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 12, @@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007640163S", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 11, @@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007307840S", "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 10, @@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007391353S", "startedAt": "2025-01-16T17:00:29.201180268Z", "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 9, @@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007445825S", "startedAt": "2025-01-16T17:00:15.77629445Z", "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 8, @@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.012020083S", "startedAt": "2025-01-16T16:59:42.744086671Z", "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 7, @@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007440092S", "startedAt": "2025-01-16T16:58:41.2155771Z", "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 6, @@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007565161S", "startedAt": "2025-01-16T16:54:51.940332781Z", "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 5, @@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.016307263S", "startedAt": "2025-01-16T16:53:19.913351957Z", "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 23, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index a28fb8789..1d89e6838 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "stats": { "totalNbTasks": 1, @@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" + "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" }, { "uid": 23, @@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.004146631S", "startedAt": "2025-01-23T11:38:57.012591321Z", "finishedAt": "2025-01-23T11:38:57.016737952Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 22, @@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.102738497S", "startedAt": "2025-01-23T11:36:22.551906856Z", "finishedAt": "2025-01-23T11:36:22.654645353Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 21, @@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.005108474S", "startedAt": "2025-01-23T11:36:04.132670526Z", "finishedAt": "2025-01-23T11:36:04.137779Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 20, @@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.027954894S", "startedAt": "2025-01-23T11:35:53.631082795Z", "finishedAt": "2025-01-23T11:35:53.659037689Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 19, @@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.006903297S", "startedAt": "2025-01-20T11:50:52.874106134Z", "finishedAt": "2025-01-20T11:50:52.881009431Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 18, @@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000481257S", "startedAt": "2025-01-20T11:48:04.92820416Z", "finishedAt": "2025-01-20T11:48:04.928685417Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 17, @@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000407005S", "startedAt": "2025-01-20T11:47:53.509403957Z", "finishedAt": "2025-01-20T11:47:53.509810962Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 16, @@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000403716S", "startedAt": "2025-01-20T11:47:48.430653005Z", "finishedAt": "2025-01-20T11:47:48.431056721Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 15, @@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000417016S", "startedAt": "2025-01-20T11:47:42.429678617Z", "finishedAt": "2025-01-20T11:47:42.430095633Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 14, @@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT12.086284842S", "startedAt": "2025-01-20T11:47:03.092181576Z", "finishedAt": "2025-01-20T11:47:15.178466418Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 13, @@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.011506614S", "startedAt": "2025-01-16T17:18:43.29334923Z", "finishedAt": "2025-01-16T17:18:43.304855844Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 12, @@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007640163S", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 11, @@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007307840S", "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 10, @@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007391353S", "startedAt": "2025-01-16T17:00:29.201180268Z", "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 9, @@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007445825S", "startedAt": "2025-01-16T17:00:15.77629445Z", "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 8, @@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.012020083S", "startedAt": "2025-01-16T16:59:42.744086671Z", "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 7, @@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007440092S", "startedAt": "2025-01-16T16:58:41.2155771Z", "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 6, @@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007565161S", "startedAt": "2025-01-16T16:54:51.940332781Z", "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 5, @@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.016307263S", "startedAt": "2025-01-16T16:53:19.913351957Z", "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 23, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index a28fb8789..1d89e6838 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "stats": { "totalNbTasks": 1, @@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" + "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" }, { "uid": 23, @@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.004146631S", "startedAt": "2025-01-23T11:38:57.012591321Z", "finishedAt": "2025-01-23T11:38:57.016737952Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 22, @@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.102738497S", "startedAt": "2025-01-23T11:36:22.551906856Z", "finishedAt": "2025-01-23T11:36:22.654645353Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 21, @@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.005108474S", "startedAt": "2025-01-23T11:36:04.132670526Z", "finishedAt": "2025-01-23T11:36:04.137779Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 20, @@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.027954894S", "startedAt": "2025-01-23T11:35:53.631082795Z", "finishedAt": "2025-01-23T11:35:53.659037689Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 19, @@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.006903297S", "startedAt": "2025-01-20T11:50:52.874106134Z", "finishedAt": "2025-01-20T11:50:52.881009431Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 18, @@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000481257S", "startedAt": "2025-01-20T11:48:04.92820416Z", "finishedAt": "2025-01-20T11:48:04.928685417Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 17, @@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000407005S", "startedAt": "2025-01-20T11:47:53.509403957Z", "finishedAt": "2025-01-20T11:47:53.509810962Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 16, @@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000403716S", "startedAt": "2025-01-20T11:47:48.430653005Z", "finishedAt": "2025-01-20T11:47:48.431056721Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 15, @@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000417016S", "startedAt": "2025-01-20T11:47:42.429678617Z", "finishedAt": "2025-01-20T11:47:42.430095633Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 14, @@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT12.086284842S", "startedAt": "2025-01-20T11:47:03.092181576Z", "finishedAt": "2025-01-20T11:47:15.178466418Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 13, @@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.011506614S", "startedAt": "2025-01-16T17:18:43.29334923Z", "finishedAt": "2025-01-16T17:18:43.304855844Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 12, @@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007640163S", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 11, @@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007307840S", "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 10, @@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007391353S", "startedAt": "2025-01-16T17:00:29.201180268Z", "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 9, @@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007445825S", "startedAt": "2025-01-16T17:00:15.77629445Z", "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 8, @@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.012020083S", "startedAt": "2025-01-16T16:59:42.744086671Z", "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 7, @@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007440092S", "startedAt": "2025-01-16T16:58:41.2155771Z", "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 6, @@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007565161S", "startedAt": "2025-01-16T16:54:51.940332781Z", "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 5, @@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.016307263S", "startedAt": "2025-01-16T16:53:19.913351957Z", "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 23, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap index 31754ca1a..341085c87 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap @@ -29,7 +29,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap index 40cb48912..15ae9c34d 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 0, @@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.111055654S", "startedAt": "2025-01-16T16:45:16.020248085Z", "finishedAt": "2025-01-16T16:45:16.131303739Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 2, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap index 40cb48912..15ae9c34d 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap @@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 0, @@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.111055654S", "startedAt": "2025-01-16T16:45:16.020248085Z", "finishedAt": "2025-01-16T16:45:16.131303739Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 2, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap index 40cb48912..15ae9c34d 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap @@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 0, @@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.111055654S", "startedAt": "2025-01-16T16:45:16.020248085Z", "finishedAt": "2025-01-16T16:45:16.131303739Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 2, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap index 7e1b7202e..04795c285 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap @@ -30,7 +30,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap index 7e1b7202e..04795c285 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap @@ -30,7 +30,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap index 31754ca1a..341085c87 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap @@ -29,7 +29,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 037aeccd6..480f85bdb 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 037aeccd6..480f85bdb 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 037aeccd6..480f85bdb 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 0796204eb..068dd0d82 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "stats": { "totalNbTasks": 1, @@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" + "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" }, { "uid": 23, @@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.004146631S", "startedAt": "2025-01-23T11:38:57.012591321Z", "finishedAt": "2025-01-23T11:38:57.016737952Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 22, @@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.102738497S", "startedAt": "2025-01-23T11:36:22.551906856Z", "finishedAt": "2025-01-23T11:36:22.654645353Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 21, @@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.005108474S", "startedAt": "2025-01-23T11:36:04.132670526Z", "finishedAt": "2025-01-23T11:36:04.137779Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 20, @@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.027954894S", "startedAt": "2025-01-23T11:35:53.631082795Z", "finishedAt": "2025-01-23T11:35:53.659037689Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 19, @@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.006903297S", "startedAt": "2025-01-20T11:50:52.874106134Z", "finishedAt": "2025-01-20T11:50:52.881009431Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 18, @@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000481257S", "startedAt": "2025-01-20T11:48:04.92820416Z", "finishedAt": "2025-01-20T11:48:04.928685417Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 17, @@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000407005S", "startedAt": "2025-01-20T11:47:53.509403957Z", "finishedAt": "2025-01-20T11:47:53.509810962Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 16, @@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000403716S", "startedAt": "2025-01-20T11:47:48.430653005Z", "finishedAt": "2025-01-20T11:47:48.431056721Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 15, @@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.000417016S", "startedAt": "2025-01-20T11:47:42.429678617Z", "finishedAt": "2025-01-20T11:47:42.430095633Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 14, @@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT12.086284842S", "startedAt": "2025-01-20T11:47:03.092181576Z", "finishedAt": "2025-01-20T11:47:15.178466418Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 13, @@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.011506614S", "startedAt": "2025-01-16T17:18:43.29334923Z", "finishedAt": "2025-01-16T17:18:43.304855844Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 12, @@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007640163S", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 11, @@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007307840S", "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 10, @@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007391353S", "startedAt": "2025-01-16T17:00:29.201180268Z", "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 9, @@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007445825S", "startedAt": "2025-01-16T17:00:15.77629445Z", "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 8, @@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.012020083S", "startedAt": "2025-01-16T16:59:42.744086671Z", "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 7, @@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007440092S", "startedAt": "2025-01-16T16:58:41.2155771Z", "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 6, @@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007565161S", "startedAt": "2025-01-16T16:54:51.940332781Z", "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 5, @@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.016307263S", "startedAt": "2025-01-16T16:53:19.913351957Z", "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 4, @@ -540,7 +540,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.087655941S", "startedAt": "2025-01-16T16:52:32.631145531Z", "finishedAt": "2025-01-16T16:52:32.718801472Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 3, @@ -565,7 +565,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.007593573S", "startedAt": "2025-01-16T16:47:53.677901409Z", "finishedAt": "2025-01-16T16:47:53.685494982Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 2, @@ -591,7 +591,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.017769760S", "startedAt": "2025-01-16T16:47:41.211587682Z", "finishedAt": "2025-01-16T16:47:41.229357442Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 1, @@ -615,7 +615,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.066095506S", "startedAt": "2025-01-16T16:47:10.217299609Z", "finishedAt": "2025-01-16T16:47:10.283395115Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" }, { "uid": 0, @@ -639,7 +639,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "duration": "PT0.111055654S", "startedAt": "2025-01-16T16:45:16.020248085Z", "finishedAt": "2025-01-16T16:45:16.131303739Z", - "batcherStoppedBecause": "unspecified" + "batchCreationComplete": "unspecified" } ], "total": 25, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 8efd2a55e..5c409891c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.14.0" + "upgradeTo": "v1.15.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilitool/src/upgrade/mod.rs b/crates/meilitool/src/upgrade/mod.rs index 82a57317c..87ce00772 100644 --- a/crates/meilitool/src/upgrade/mod.rs +++ b/crates/meilitool/src/upgrade/mod.rs @@ -49,15 +49,10 @@ impl OfflineUpgrade { const LAST_SUPPORTED_UPGRADE_TO_VERSION: &str = "1.12.7"; let upgrade_list = [ - ( - v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, - "1", - "10", - "0", - ), - (v1_10_to_v1_11, "1", "11", "0"), - (v1_11_to_v1_12, "1", "12", "0"), - (v1_12_to_v1_12_3, "1", "12", "3"), + (v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, 1, 10, 0), + (v1_10_to_v1_11, 1, 11, 0), + (v1_11_to_v1_12, 1, 12, 0), + (v1_12_to_v1_12_3, 1, 12, 3), ]; let no_upgrade: usize = upgrade_list.len(); @@ -95,13 +90,8 @@ impl OfflineUpgrade { if start_at == no_upgrade { println!("No upgrade operation to perform, writing VERSION file"); - create_version_file( - &self.db_path, - &target_major.to_string(), - &target_minor.to_string(), - &target_patch.to_string(), - ) - .context("while writing VERSION file after the upgrade")?; + create_version_file(&self.db_path, target_major, target_minor, target_patch) + .context("while writing VERSION file after the upgrade")?; println!("Success"); return Ok(()); } diff --git a/crates/milli/src/constants.rs b/crates/milli/src/constants.rs index 39b449661..dc88bdb37 100644 --- a/crates/milli/src/constants.rs +++ b/crates/milli/src/constants.rs @@ -1,6 +1,13 @@ -pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); -pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); -pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); +pub const VERSION_MAJOR: u32 = parse_u32(env!("CARGO_PKG_VERSION_MAJOR")); +pub const VERSION_MINOR: u32 = parse_u32(env!("CARGO_PKG_VERSION_MINOR")); +pub const VERSION_PATCH: u32 = parse_u32(env!("CARGO_PKG_VERSION_PATCH")); + +const fn parse_u32(s: &str) -> u32 { + match u32::from_str_radix(s, 10) { + Ok(version) => version, + Err(_) => panic!("could not parse as u32"), + } +} pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors"; pub const RESERVED_GEO_FIELD_NAME: &str = "_geo"; diff --git a/crates/milli/src/disabled_typos_terms.rs b/crates/milli/src/disabled_typos_terms.rs new file mode 100644 index 000000000..3a0d0c0f5 --- /dev/null +++ b/crates/milli/src/disabled_typos_terms.rs @@ -0,0 +1,50 @@ +use heed::{ + types::{SerdeJson, Str}, + RoTxn, RwTxn, +}; +use serde::{Deserialize, Serialize}; + +use crate::{index::main_key, Index}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase")] +pub struct DisabledTyposTerms { + pub disable_on_numbers: bool, +} + +impl Index { + pub fn disabled_typos_terms(&self, txn: &RoTxn<'_>) -> heed::Result { + self.main + .remap_types::>() + .get(txn, main_key::DISABLED_TYPOS_TERMS) + .map(|option| option.unwrap_or_default()) + } + + pub(crate) fn put_disabled_typos_terms( + &self, + txn: &mut RwTxn<'_>, + disabled_typos_terms: &DisabledTyposTerms, + ) -> heed::Result<()> { + self.main.remap_types::>().put( + txn, + main_key::DISABLED_TYPOS_TERMS, + disabled_typos_terms, + )?; + + Ok(()) + } + + pub(crate) fn delete_disabled_typos_terms(&self, txn: &mut RwTxn<'_>) -> heed::Result<()> { + self.main + .remap_types::>() + .delete(txn, main_key::DISABLED_TYPOS_TERMS)?; + Ok(()) + } +} + +impl DisabledTyposTerms { + pub fn is_exact(&self, word: &str) -> bool { + // If disable_on_numbers is true, we disable the word if it contains only numbers or punctuation + self.disable_on_numbers && word.chars().all(|c| c.is_numeric() || c.is_ascii_punctuation()) + } +} diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 948d0fb0d..d0cd5c862 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -3,8 +3,8 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs::File; use std::path::Path; -use heed::{types::*, DatabaseStat, WithoutTls}; -use heed::{CompactionOption, Database, RoTxn, RwTxn, Unspecified}; +use heed::types::*; +use heed::{CompactionOption, Database, DatabaseStat, RoTxn, RwTxn, Unspecified, WithoutTls}; use indexmap::IndexMap; use roaring::RoaringBitmap; use rstar::RTree; @@ -78,6 +78,7 @@ pub mod main_key { pub const FACET_SEARCH: &str = "facet_search"; pub const PREFIX_SEARCH: &str = "prefix_search"; pub const DOCUMENTS_STATS: &str = "documents_stats"; + pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms"; } pub mod db_name { @@ -107,6 +108,7 @@ pub mod db_name { pub const VECTOR_ARROY: &str = "vector-arroy"; pub const DOCUMENTS: &str = "documents"; } +const NUMBER_OF_DBS: u32 = 25; #[derive(Clone)] pub struct Index { @@ -186,7 +188,7 @@ impl Index { ) -> Result { use db_name::*; - options.max_dbs(25); + options.max_dbs(NUMBER_OF_DBS); let env = unsafe { options.open(path) }?; let mut wtxn = env.write_txn()?; @@ -261,11 +263,7 @@ impl Index { if this.get_version(&wtxn)?.is_none() && creation { this.put_version( &mut wtxn, - ( - constants::VERSION_MAJOR.parse().unwrap(), - constants::VERSION_MINOR.parse().unwrap(), - constants::VERSION_PATCH.parse().unwrap(), - ), + (constants::VERSION_MAJOR, constants::VERSION_MINOR, constants::VERSION_PATCH), )?; } wtxn.commit()?; @@ -284,6 +282,76 @@ impl Index { Self::new_with_creation_dates(options, path, now, now, creation) } + /// Attempts to rollback the index at `path` to the version specified by `requested_version`. + pub fn rollback>( + mut options: heed::EnvOpenOptions, + path: P, + requested_version: (u32, u32, u32), + ) -> Result { + options.max_dbs(NUMBER_OF_DBS); + + // optimistically check if the index is already at the requested version. + let env = unsafe { options.open(path.as_ref()) }?; + let rtxn = env.read_txn()?; + let Some(main) = env.database_options().name(db_name::MAIN).open(&rtxn)? else { + return Err(crate::Error::InternalError(crate::InternalError::DatabaseMissingEntry { + db_name: db_name::MAIN, + key: None, + })); + }; + let rollback_version = + main.remap_types::().get(&rtxn, main_key::VERSION_KEY)?; + if rollback_version == Some(requested_version) { + return Ok(RollbackOutcome::NoRollback); + } + + // explicitly drop the environment before reopening it. + drop(rtxn); + drop(env); + + // really need to rollback then... + unsafe { options.flags(heed::EnvFlags::PREV_SNAPSHOT) }; + let env = unsafe { options.open(path) }?; + let mut wtxn = env.write_txn()?; + let Some(main) = env.database_options().name(db_name::MAIN).open(&wtxn)? else { + return Err(crate::Error::InternalError(crate::InternalError::DatabaseMissingEntry { + db_name: db_name::MAIN, + key: None, + })); + }; + + let main = main.remap_key_type::(); + + let Some(rollback_version) = + main.remap_data_type::().get(&wtxn, main_key::VERSION_KEY)? + else { + return Ok(RollbackOutcome::VersionMismatch { + requested_version, + rollback_version: None, + }); + }; + + if requested_version != rollback_version { + return Ok(RollbackOutcome::VersionMismatch { + requested_version, + rollback_version: Some(rollback_version), + }); + } + + // this is a bit of a trick to force a change in the index + // which is necessary to actually discard the next snapshot, replacing it with this transaction. + let now = time::OffsetDateTime::now_utc(); + main.remap_data_type::>().put( + &mut wtxn, + main_key::UPDATED_AT_KEY, + &OffsetDateTime(now), + )?; + + wtxn.commit()?; + + Ok(RollbackOutcome::Rollback) + } + fn set_creation_dates( env: &heed::Env, main: Database, @@ -374,7 +442,7 @@ impl Index { } /// Get the version of the database. `None` if it was never set. - pub(crate) fn get_version(&self, rtxn: &RoTxn<'_>) -> heed::Result> { + pub fn get_version(&self, rtxn: &RoTxn<'_>) -> heed::Result> { self.main.remap_types::().get(rtxn, main_key::VERSION_KEY) } @@ -1864,1397 +1932,39 @@ pub enum PrefixSearch { Disabled, } +#[derive(Debug)] +pub enum RollbackOutcome { + VersionMismatch { + requested_version: (u32, u32, u32), + rollback_version: Option<(u32, u32, u32)>, + }, + Rollback, + NoRollback, +} + +impl RollbackOutcome { + pub fn succeeded(&self) -> bool { + matches!(self, RollbackOutcome::Rollback | RollbackOutcome::NoRollback) + } +} + +impl std::fmt::Display for RollbackOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RollbackOutcome::VersionMismatch { requested_version, rollback_version: Some(rollback_version) } => write!(f, "cannot rollback to the requested version\n - note: requested version is v{}.{}.{}\n - note: only possible to rollback to v{}.{}.{}", + requested_version.0, requested_version.1, requested_version.2, rollback_version.0, rollback_version.1, rollback_version.2), + RollbackOutcome::VersionMismatch { requested_version, rollback_version: None } => write!(f, "cannot rollback to the requested version\n - note: requested version is v{}.{}.{}\n - note: only possible to rollback to an unknown version", + requested_version.0, requested_version.1, requested_version.2), + RollbackOutcome::Rollback => f.write_str("rollback complete"), + RollbackOutcome::NoRollback => f.write_str("no rollback necessary"), + } + } +} + #[derive(Serialize, Deserialize)] #[serde(transparent)] struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] time::OffsetDateTime); #[cfg(test)] -pub(crate) mod tests { - use std::collections::HashSet; - use std::ops::Deref; - - use big_s::S; - use bumpalo::Bump; - use heed::{EnvOpenOptions, RwTxn}; - use maplit::btreemap; - use memmap2::Mmap; - use tempfile::TempDir; - - use crate::constants::RESERVED_GEO_FIELD_NAME; - use crate::error::{Error, InternalError}; - use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; - use crate::progress::Progress; - use crate::update::new::indexer; - use crate::update::settings::InnerIndexSettings; - use crate::update::{ - self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings, - }; - use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; - use crate::vector::EmbeddingConfigs; - use crate::{ - db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, - }; - - pub(crate) struct TempIndex { - pub inner: Index, - pub indexer_config: IndexerConfig, - pub index_documents_config: IndexDocumentsConfig, - _tempdir: TempDir, - } - - impl Deref for TempIndex { - type Target = Index; - - fn deref(&self) -> &Self::Target { - &self.inner - } - } - - impl TempIndex { - /// Creates a temporary index - pub fn new_with_map_size(size: usize) -> Self { - let options = EnvOpenOptions::new(); - let mut options = options.read_txn_without_tls(); - options.map_size(size); - let _tempdir = TempDir::new_in(".").unwrap(); - let inner = Index::new(options, _tempdir.path(), true).unwrap(); - let indexer_config = IndexerConfig::default(); - let index_documents_config = IndexDocumentsConfig::default(); - Self { inner, indexer_config, index_documents_config, _tempdir } - } - /// Creates a temporary index, with a default `4096 * 2000` size. This should be enough for - /// most tests. - pub fn new() -> Self { - Self::new_with_map_size(4096 * 2000) - } - - pub fn add_documents_using_wtxn<'t>( - &'t self, - wtxn: &mut RwTxn<'t>, - documents: Mmap, - ) -> Result<(), crate::error::Error> { - let indexer_config = &self.indexer_config; - let pool = &indexer_config.thread_pool; - - let rtxn = self.inner.read_txn()?; - let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?; - let mut new_fields_ids_map = db_fields_ids_map.clone(); - - let embedders = - InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs; - let mut indexer = indexer::DocumentOperation::new(); - match self.index_documents_config.update_method { - IndexDocumentsMethod::ReplaceDocuments => { - indexer.replace_documents(&documents).unwrap() - } - IndexDocumentsMethod::UpdateDocuments => { - indexer.update_documents(&documents).unwrap() - } - } - - let indexer_alloc = Bump::new(); - let (document_changes, operation_stats, primary_key) = indexer.into_changes( - &indexer_alloc, - &self.inner, - &rtxn, - None, - &mut new_fields_ids_map, - &|| false, - Progress::default(), - )?; - - if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { - return Err(error.into()); - } - - pool.install(|| { - indexer::index( - wtxn, - &self.inner, - &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), - indexer_config.grenad_parameters(), - &db_fields_ids_map, - new_fields_ids_map, - primary_key, - &document_changes, - embedders, - &|| false, - &Progress::default(), - ) - }) - .unwrap()?; - - Ok(()) - } - - pub fn add_documents(&self, documents: Mmap) -> Result<(), crate::error::Error> { - let mut wtxn = self.write_txn().unwrap(); - self.add_documents_using_wtxn(&mut wtxn, documents)?; - wtxn.commit().unwrap(); - Ok(()) - } - - pub fn update_settings( - &self, - update: impl Fn(&mut Settings<'_, '_, '_>), - ) -> Result<(), crate::error::Error> { - let mut wtxn = self.write_txn().unwrap(); - self.update_settings_using_wtxn(&mut wtxn, update)?; - wtxn.commit().unwrap(); - Ok(()) - } - - pub fn update_settings_using_wtxn<'t>( - &'t self, - wtxn: &mut RwTxn<'t>, - update: impl Fn(&mut Settings<'_, '_, '_>), - ) -> Result<(), crate::error::Error> { - let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config); - update(&mut builder); - builder.execute(drop, || false)?; - Ok(()) - } - - pub fn delete_documents_using_wtxn<'t>( - &'t self, - wtxn: &mut RwTxn<'t>, - external_document_ids: Vec, - ) -> Result<(), crate::error::Error> { - let indexer_config = &self.indexer_config; - let pool = &indexer_config.thread_pool; - - let rtxn = self.inner.read_txn()?; - let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?; - let mut new_fields_ids_map = db_fields_ids_map.clone(); - - let embedders = - InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs; - - let mut indexer = indexer::DocumentOperation::new(); - let external_document_ids: Vec<_> = - external_document_ids.iter().map(AsRef::as_ref).collect(); - indexer.delete_documents(external_document_ids.as_slice()); - - let indexer_alloc = Bump::new(); - let (document_changes, operation_stats, primary_key) = indexer.into_changes( - &indexer_alloc, - &self.inner, - &rtxn, - None, - &mut new_fields_ids_map, - &|| false, - Progress::default(), - )?; - - if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { - return Err(error.into()); - } - - pool.install(|| { - indexer::index( - wtxn, - &self.inner, - &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), - indexer_config.grenad_parameters(), - &db_fields_ids_map, - new_fields_ids_map, - primary_key, - &document_changes, - embedders, - &|| false, - &Progress::default(), - ) - }) - .unwrap()?; - - Ok(()) - } - - pub fn delete_documents(&self, external_document_ids: Vec) { - let mut wtxn = self.write_txn().unwrap(); - - self.delete_documents_using_wtxn(&mut wtxn, external_document_ids).unwrap(); - - wtxn.commit().unwrap(); - } - - pub fn delete_document(&self, external_document_id: &str) { - self.delete_documents(vec![external_document_id.to_string()]) - } - } - - #[test] - fn aborting_indexation() { - use std::sync::atomic::AtomicBool; - use std::sync::atomic::Ordering::Relaxed; - - let index = TempIndex::new(); - let mut wtxn = index.inner.write_txn().unwrap(); - let should_abort = AtomicBool::new(false); - - let indexer_config = &index.indexer_config; - let pool = &indexer_config.thread_pool; - - let rtxn = index.inner.read_txn().unwrap(); - let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap(); - let mut new_fields_ids_map = db_fields_ids_map.clone(); - - let embedders = EmbeddingConfigs::default(); - let mut indexer = indexer::DocumentOperation::new(); - let payload = documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "bob", "age": 20 }, - { "id": 2, "name": "bob", "age": 20 }, - ]); - indexer.replace_documents(&payload).unwrap(); - - let indexer_alloc = Bump::new(); - let (document_changes, _operation_stats, primary_key) = indexer - .into_changes( - &indexer_alloc, - &index.inner, - &rtxn, - None, - &mut new_fields_ids_map, - &|| false, - Progress::default(), - ) - .unwrap(); - - should_abort.store(true, Relaxed); - - let err = pool - .install(|| { - indexer::index( - &mut wtxn, - &index.inner, - &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), - indexer_config.grenad_parameters(), - &db_fields_ids_map, - new_fields_ids_map, - primary_key, - &document_changes, - embedders, - &|| should_abort.load(Relaxed), - &Progress::default(), - ) - }) - .unwrap() - .unwrap_err(); - - assert!(matches!(err, Error::InternalError(InternalError::AbortedIndexation))); - } - - #[test] - fn initial_field_distribution() { - let index = TempIndex::new(); - index - .add_documents(documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "bob", "age": 20 }, - { "id": 2, "name": "bob", "age": 20 }, - ])) - .unwrap(); - - db_snap!(index, field_distribution, @r###" - age 1 | - id 2 | - name 2 | - "###); - - db_snap!(index, word_docids, - @r###" - 1 [0, ] - 2 [1, ] - 20 [1, ] - bob [1, ] - kevin [0, ] - "### - ); - - // we add all the documents a second time. we are supposed to get the same - // field_distribution in the end - index - .add_documents(documents!([ - { "id": 1, "name": "kevin" }, - { "id": 2, "name": "bob", "age": 20 }, - { "id": 2, "name": "bob", "age": 20 }, - ])) - .unwrap(); - - db_snap!(index, field_distribution, - @r###" - age 1 | - id 2 | - name 2 | - "### - ); - - // then we update a document by removing one field and another by adding one field - index - .add_documents(documents!([ - { "id": 1, "name": "kevin", "has_dog": true }, - { "id": 2, "name": "bob" } - ])) - .unwrap(); - - db_snap!(index, field_distribution, - @r###" - has_dog 1 | - id 2 | - name 2 | - "### - ); - } - - #[test] - fn put_and_retrieve_disable_typo() { - let index = TempIndex::new(); - let mut txn = index.write_txn().unwrap(); - // default value is true - assert!(index.authorize_typos(&txn).unwrap()); - // set to false - index.put_authorize_typos(&mut txn, false).unwrap(); - txn.commit().unwrap(); - - let txn = index.read_txn().unwrap(); - assert!(!index.authorize_typos(&txn).unwrap()); - } - - #[test] - fn set_min_word_len_for_typos() { - let index = TempIndex::new(); - let mut txn = index.write_txn().unwrap(); - - assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_ONE_TYPO); - assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_TWO_TYPOS); - - index.put_min_word_len_one_typo(&mut txn, 3).unwrap(); - index.put_min_word_len_two_typos(&mut txn, 15).unwrap(); - - txn.commit().unwrap(); - - let txn = index.read_txn().unwrap(); - assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), 3); - assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), 15); - } - - #[test] - fn add_documents_and_set_searchable_fields() { - let index = TempIndex::new(); - index - .add_documents(documents!([ - { "id": 1, "doggo": "kevin" }, - { "id": 2, "doggo": { "name": "bob", "age": 20 } }, - { "id": 3, "name": "jean", "age": 25 }, - ])) - .unwrap(); - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("doggo"), S("name")]); - }) - .unwrap(); - - // ensure we get the right real searchable fields + user defined searchable fields - let rtxn = index.read_txn().unwrap(); - - let real = index.searchable_fields(&rtxn).unwrap(); - assert_eq!(real, &["doggo", "name", "doggo.name", "doggo.age"]); - - let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); - assert_eq!(user_defined, &["doggo", "name"]); - } - - #[test] - fn set_searchable_fields_and_add_documents() { - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("doggo"), S("name")]); - }) - .unwrap(); - - // ensure we get the right real searchable fields + user defined searchable fields - let rtxn = index.read_txn().unwrap(); - - let real = index.searchable_fields(&rtxn).unwrap(); - assert!(real.is_empty()); - let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); - assert_eq!(user_defined, &["doggo", "name"]); - - index - .add_documents(documents!([ - { "id": 1, "doggo": "kevin" }, - { "id": 2, "doggo": { "name": "bob", "age": 20 } }, - { "id": 3, "name": "jean", "age": 25 }, - ])) - .unwrap(); - - // ensure we get the right real searchable fields + user defined searchable fields - let rtxn = index.read_txn().unwrap(); - - let real = index.searchable_fields(&rtxn).unwrap(); - assert_eq!(real, &["doggo", "name", "doggo.name", "doggo.age"]); - - let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); - assert_eq!(user_defined, &["doggo", "name"]); - } - - #[test] - fn test_basic_geo_bounding_box() { - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - RESERVED_GEO_FIELD_NAME.to_string(), - )]); - }) - .unwrap(); - index - .add_documents(documents!([ - { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": "0" } }, - { "id": 1, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": "-175" } }, - { "id": 2, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": 175 } }, - { "id": 3, RESERVED_GEO_FIELD_NAME: { "lat": 85, "lng": 0 } }, - { "id": 4, RESERVED_GEO_FIELD_NAME: { "lat": "-85", "lng": "0" } }, - ])) - .unwrap(); - - // ensure we get the right real searchable fields + user defined searchable fields - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - - // exact match a document - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([0, 0], [0, 0])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0]>"); - - // match a document in the middle of the rectangle - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([10, 10], [-10, -10])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0]>"); - - // select everything - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([90, 180], [-90, -180])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1, 2, 3, 4]>"); - - // go on the edge of the longitude - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([0, -170], [0, 180])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[1]>"); - - // go on the other edge of the longitude - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([0, -180], [0, 170])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[2]>"); - - // wrap around the longitude - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([0, -170], [0, 170])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[1, 2]>"); - - // go on the edge of the latitude - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([90, 0], [80, 0])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[3]>"); - - // go on the edge of the latitude - let search_result = search - .filter(Filter::from_str("_geoBoundingBox([-80, 0], [-90, 0])").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[4]>"); - - // the requests that don't make sense - - // try to wrap around the latitude - let error = search - .filter(Filter::from_str("_geoBoundingBox([-80, 0], [80, 0])").unwrap().unwrap()) - .execute() - .unwrap_err(); - insta::assert_snapshot!( - error, - @r###" - The top latitude `-80` is below the bottom latitude `80`. - 32:33 _geoBoundingBox([-80, 0], [80, 0]) - "### - ); - - // send a top latitude lower than the bottow latitude - let error = search - .filter(Filter::from_str("_geoBoundingBox([-10, 0], [10, 0])").unwrap().unwrap()) - .execute() - .unwrap_err(); - insta::assert_snapshot!( - error, - @r###" - The top latitude `-10` is below the bottom latitude `10`. - 32:33 _geoBoundingBox([-10, 0], [10, 0]) - "### - ); - } - - #[test] - fn test_contains() { - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - "doggo".to_string(), - )]); - }) - .unwrap(); - index - .add_documents(documents!([ - { "id": 0, "doggo": "kefir" }, - { "id": 1, "doggo": "kefirounet" }, - { "id": 2, "doggo": "kefkef" }, - { "id": 3, "doggo": "fifir" }, - { "id": 4, "doggo": "boubou" }, - { "id": 5 }, - ])) - .unwrap(); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let search_result = search - .filter(Filter::from_str("doggo CONTAINS kefir").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1]>"); - let mut search = index.search(&rtxn); - let search_result = search - .filter(Filter::from_str("doggo CONTAINS KEF").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1, 2]>"); - let mut search = index.search(&rtxn); - let search_result = search - .filter(Filter::from_str("doggo NOT CONTAINS fir").unwrap().unwrap()) - .execute() - .unwrap(); - insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[2, 4, 5]>"); - } - - #[test] - fn replace_documents_external_ids_and_soft_deletion_check() { - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_primary_key("id".to_owned()); - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - "doggo".to_string(), - )]); - }) - .unwrap(); - - let mut docs = vec![]; - for i in 0..4 { - docs.push(serde_json::json!( - { "id": i, "doggo": i } - )); - } - index.add_documents(documents!(docs)).unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); - db_snap!(index, external_documents_ids, 1, @r###" - docids: - 0 0 - 1 1 - 2 2 - 3 3 - "###); - db_snap!(index, facet_id_f64_docids, 1, @r###" - 1 0 0 1 [0, ] - 1 0 1 1 [1, ] - 1 0 2 1 [2, ] - 1 0 3 1 [3, ] - "###); - - let mut docs = vec![]; - for i in 0..3 { - docs.push(serde_json::json!( - { "id": i, "doggo": i + 1 } - )); - } - index.add_documents(documents!(docs)).unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 0 0 - 1 1 - 2 2 - 3 3 - "###); - db_snap!(index, facet_id_f64_docids, 2, @r###" - 1 0 1 1 [0, ] - 1 0 2 1 [1, ] - 1 0 3 1 [2, 3, ] - "###); - - index - .add_documents(documents!([{ "id": 3, "doggo": 4 }, { "id": 3, "doggo": 5 },{ "id": 3, "doggo": 4 }])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); - db_snap!(index, external_documents_ids, 3, @r###" - docids: - 0 0 - 1 1 - 2 2 - 3 3 - "###); - db_snap!(index, facet_id_f64_docids, 3, @r###" - 1 0 1 1 [0, ] - 1 0 2 1 [1, ] - 1 0 3 1 [2, ] - 1 0 4 1 [3, ] - "###); - - index - .update_settings(|settings| { - settings.set_distinct_field("id".to_owned()); - }) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); - db_snap!(index, external_documents_ids, 3, @r###" - docids: - 0 0 - 1 1 - 2 2 - 3 3 - "###); - db_snap!(index, facet_id_f64_docids, 3, @r###" - 0 0 0 1 [0, ] - 0 0 1 1 [1, ] - 0 0 2 1 [2, ] - 0 0 3 1 [3, ] - 1 0 1 1 [0, ] - 1 0 2 1 [1, ] - 1 0 3 1 [2, ] - 1 0 4 1 [3, ] - "###); - } - - #[test] - fn bug_3021_first() { - // https://github.com/meilisearch/meilisearch/issues/3021 - let mut index = TempIndex::new(); - index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; - - index - .update_settings(|settings| { - settings.set_primary_key("primary_key".to_owned()); - }) - .unwrap(); - - index - .add_documents(documents!([ - { "primary_key": 38 }, - { "primary_key": 34 } - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - db_snap!(index, external_documents_ids, 1, @r###" - docids: - 34 1 - 38 0 - "###); - - index.delete_document("34"); - - db_snap!(index, documents_ids, @"[0, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 38 0 - "###); - - index - .update_settings(|s| { - s.set_searchable_fields(vec![]); - }) - .unwrap(); - - // The key point of the test is to verify that the external documents ids - // do not contain any entry for previously soft-deleted document ids - db_snap!(index, documents_ids, @"[0, ]"); - db_snap!(index, external_documents_ids, 3, @r###" - docids: - 38 0 - "###); - - // So that this document addition works correctly now. - // It would be wrongly interpreted as a replacement before - index.add_documents(documents!({ "primary_key": 34 })).unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - db_snap!(index, external_documents_ids, 4, @r###" - docids: - 34 1 - 38 0 - "###); - - // We do the test again, but deleting the document with id 0 instead of id 1 now - index.delete_document("38"); - - db_snap!(index, documents_ids, @"[1, ]"); - db_snap!(index, external_documents_ids, 5, @r###" - docids: - 34 1 - "###); - - index - .update_settings(|s| { - s.set_searchable_fields(vec!["primary_key".to_owned()]); - }) - .unwrap(); - - db_snap!(index, documents_ids, @"[1, ]"); - db_snap!(index, external_documents_ids, 6, @r###" - docids: - 34 1 - "###); - - // And adding lots of documents afterwards instead of just one. - // These extra subtests don't add much, but it's better than nothing. - index - .add_documents(documents!([ - { "primary_key": 38 }, - { "primary_key": 39 }, - { "primary_key": 41 }, - { "primary_key": 40 }, - { "primary_key": 41 }, - { "primary_key": 42 }, - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, 4, 5, ]"); - db_snap!(index, external_documents_ids, 7, @r###" - docids: - 34 1 - 38 0 - 39 2 - 40 4 - 41 3 - 42 5 - "###); - } - - #[test] - fn simple_delete() { - let mut index = TempIndex::new(); - index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; - index - .add_documents(documents!([ - { "id": 30 }, - { "id": 34 } - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - db_snap!(index, external_documents_ids, 1, @r###" - docids: - 30 0 - 34 1"###); - - index.delete_document("34"); - - db_snap!(index, documents_ids, @"[0, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 30 0 - "###); - } - - #[test] - fn bug_3021_second() { - // https://github.com/meilisearch/meilisearch/issues/3021 - let mut index = TempIndex::new(); - index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; - - index - .update_settings(|settings| { - settings.set_primary_key("primary_key".to_owned()); - }) - .unwrap(); - - index - .add_documents(documents!([ - { "primary_key": 30 }, - { "primary_key": 34 } - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - db_snap!(index, external_documents_ids, 1, @r###" - docids: - 30 0 - 34 1 - "###); - - index.delete_document("34"); - - db_snap!(index, documents_ids, @"[0, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 30 0 - "###); - - index - .update_settings(|s| { - s.set_searchable_fields(vec![]); - }) - .unwrap(); - - // The key point of the test is to verify that the external documents ids - // do not contain any entry for previously soft-deleted document ids - db_snap!(index, documents_ids, @"[0, ]"); - db_snap!(index, external_documents_ids, 3, @r###" - docids: - 30 0 - "###); - - // So that when we add a new document - index.add_documents(documents!({ "primary_key": 35, "b": 2 })).unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - // The external documents ids don't have several external ids pointing to the same - // internal document id - db_snap!(index, external_documents_ids, 4, @r###" - docids: - 30 0 - 35 1 - "###); - - // And when we add 34 again, we don't replace document 35 - index.add_documents(documents!({ "primary_key": 34, "a": 1 })).unwrap(); - - // And document 35 still exists, is not deleted - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, 5, @r###" - docids: - 30 0 - 34 2 - 35 1 - "###); - - let rtxn = index.read_txn().unwrap(); - let (_docid, obkv) = index.documents(&rtxn, [0]).unwrap()[0]; - let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); - insta::assert_debug_snapshot!(json, @r###" - { - "primary_key": Number(30), - } - "###); - - // Furthermore, when we retrieve document 34, it is not the result of merging 35 with 34 - let (_docid, obkv) = index.documents(&rtxn, [2]).unwrap()[0]; - let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); - insta::assert_debug_snapshot!(json, @r###" - { - "primary_key": Number(34), - "a": Number(1), - } - "###); - - drop(rtxn); - - // Add new documents again - index - .add_documents( - documents!([{ "primary_key": 37 }, { "primary_key": 38 }, { "primary_key": 39 }]), - ) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, 3, 4, 5, ]"); - db_snap!(index, external_documents_ids, 6, @r###" - docids: - 30 0 - 34 2 - 35 1 - 37 3 - 38 4 - 39 5 - "###); - } - - #[test] - fn bug_3021_third() { - // https://github.com/meilisearch/meilisearch/issues/3021 - let mut index = TempIndex::new(); - index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; - - index - .update_settings(|settings| { - settings.set_primary_key("primary_key".to_owned()); - }) - .unwrap(); - - index - .add_documents(documents!([ - { "primary_key": 3 }, - { "primary_key": 4 }, - { "primary_key": 5 } - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, 1, @r###" - docids: - 3 0 - 4 1 - 5 2 - "###); - - index.delete_document("3"); - - db_snap!(index, documents_ids, @"[1, 2, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 4 1 - 5 2 - "###); - - index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap(); - - db_snap!(index, documents_ids, @"[1, 2, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 4 1 - 5 2 - "###); - - index - .add_documents(documents!([ - { "primary_key": "3" }, - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, 2, @r###" - docids: - 3 0 - 4 1 - 5 2 - "###); - } - - #[test] - fn bug_3021_fourth() { - // https://github.com/meilisearch/meilisearch/issues/3021 - let mut index = TempIndex::new(); - index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; - - index - .update_settings(|settings| { - settings.set_primary_key("primary_key".to_owned()); - }) - .unwrap(); - - index - .add_documents(documents!([ - { "primary_key": 11 }, - { "primary_key": 4 }, - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, ]"); - db_snap!(index, external_documents_ids, @r###" - docids: - 11 0 - 4 1 - "###); - db_snap!(index, fields_ids_map, @r###" - 0 primary_key | - "###); - db_snap!(index, searchable_fields, @r###"["primary_key"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 0 0 | - "###); - - index - .add_documents(documents!([ - { "primary_key": 4, "a": 0 }, - { "primary_key": 1 }, - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, @r###" - docids: - 1 2 - 11 0 - 4 1 - "###); - db_snap!(index, fields_ids_map, @r###" - 0 primary_key | - 1 a | - "###); - db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 0 0 | - 1 0 | - "###); - - index.delete_documents(Default::default()); - - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, @r###" - docids: - 1 2 - 11 0 - 4 1 - "###); - db_snap!(index, fields_ids_map, @r###" - 0 primary_key | - 1 a | - "###); - db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 0 0 | - 1 0 | - "###); - - index - .add_documents(documents!([ - { "primary_key": 4, "a": 1 }, - { "primary_key": 1, "a": 0 }, - ])) - .unwrap(); - - db_snap!(index, documents_ids, @"[0, 1, 2, ]"); - db_snap!(index, external_documents_ids, @r###" - docids: - 1 2 - 11 0 - 4 1 - "###); - db_snap!(index, fields_ids_map, @r###" - 0 primary_key | - 1 a | - "###); - db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 0 0 | - 1 0 | - "###); - - let rtxn = index.read_txn().unwrap(); - let search = Search::new(&rtxn, &index); - let SearchResult { - matching_words: _, - candidates: _, - document_scores: _, - mut documents_ids, - degraded: _, - used_negative_operator: _, - } = search.execute().unwrap(); - let primary_key_id = index.fields_ids_map(&rtxn).unwrap().id("primary_key").unwrap(); - documents_ids.sort_unstable(); - let docs = index.documents(&rtxn, documents_ids).unwrap(); - let mut all_ids = HashSet::new(); - for (_docid, obkv) in docs { - let id = obkv.get(primary_key_id).unwrap(); - assert!(all_ids.insert(id)); - } - } - - #[test] - fn bug_3007() { - // https://github.com/meilisearch/meilisearch/issues/3007 - - use crate::error::{GeoError, UserError}; - let index = TempIndex::new(); - - // Given is an index with a geo field NOT contained in the sortable_fields of the settings - index - .update_settings(|settings| { - settings.set_primary_key("id".to_string()); - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - RESERVED_GEO_FIELD_NAME.to_string(), - )]); - }) - .unwrap(); - - // happy path - index - .add_documents( - documents!({ "id" : 5, RESERVED_GEO_FIELD_NAME: {"lat": 12.0, "lng": 11.0}}), - ) - .unwrap(); - - db_snap!(index, geo_faceted_documents_ids); - - // both are unparseable, we expect GeoError::BadLatitudeAndLongitude - let err1 = index - .add_documents( - documents!({ "id" : 6, RESERVED_GEO_FIELD_NAME: {"lat": "unparseable", "lng": "unparseable"}}), - ) - .unwrap_err(); - match err1 { - Error::UserError(UserError::InvalidGeoField(err)) => match *err { - GeoError::BadLatitudeAndLongitude { .. } => (), - otherwise => { - panic!("err1 is not a BadLatitudeAndLongitude error but rather a {otherwise:?}") - } - }, - _ => panic!("err1 is not a BadLatitudeAndLongitude error but rather a {err1:?}"), - } - - db_snap!(index, geo_faceted_documents_ids); // ensure that no more document was inserted - } - - #[test] - fn unexpected_extra_fields_in_geo_field() { - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_primary_key("id".to_string()); - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - RESERVED_GEO_FIELD_NAME.to_string(), - )]); - }) - .unwrap(); - - let err = index - .add_documents( - documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best" }}), - ) - .unwrap_err(); - insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"doggo":"are the best"}`."###); - - db_snap!(index, geo_faceted_documents_ids); // ensure that no documents were inserted - - // multiple fields and complex values - let err = index - .add_documents( - documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best", "and": { "all": ["cats", { "are": "beautiful" } ] } } }), - ) - .unwrap_err(); - insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"and":{"all":["cats",{"are":"beautiful"}]},"doggo":"are the best"}`."###); - - db_snap!(index, geo_faceted_documents_ids); // ensure that no documents were inserted - } - - #[test] - fn swapping_searchable_attributes() { - // See https://github.com/meilisearch/meilisearch/issues/4484 - - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("name")]); - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - "age".to_string(), - )]); - }) - .unwrap(); - - index - .add_documents(documents!({ "id": 1, "name": "Many", "age": 28, "realName": "Maxime" })) - .unwrap(); - db_snap!(index, fields_ids_map, @r###" - 0 id | - 1 name | - 2 age | - 3 realName | - "###); - db_snap!(index, searchable_fields, @r###"["name"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 1 0 | - "###); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("name"), S("realName")]); - settings.set_filterable_fields(vec![FilterableAttributesRule::Field( - "age".to_string(), - )]); - }) - .unwrap(); - - // The order of the field id map shouldn't change - db_snap!(index, fields_ids_map, @r###" - 0 id | - 1 name | - 2 age | - 3 realName | - "###); - db_snap!(index, searchable_fields, @r###"["name", "realName"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 1 0 | - 3 1 | - "###); - } - - #[test] - fn attribute_weights_after_swapping_searchable_attributes() { - // See https://github.com/meilisearch/meilisearch/issues/4484 - - let index = TempIndex::new(); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("name"), S("beverage")]); - }) - .unwrap(); - - index - .add_documents(documents!([ - { "id": 0, "name": "kefir", "beverage": "water" }, - { "id": 1, "name": "tamo", "beverage": "kefir" } - ])) - .unwrap(); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let results = search.query("kefir").execute().unwrap(); - - // We should find kefir the dog first - insta::assert_debug_snapshot!(results.documents_ids, @r###" - [ - 0, - 1, - ] - "###); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("beverage"), S("name")]); - }) - .unwrap(); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let results = search.query("kefir").execute().unwrap(); - - // We should find tamo first - insta::assert_debug_snapshot!(results.documents_ids, @r###" - [ - 1, - 0, - ] - "###); - } - - #[test] - fn vectors_are_never_indexed_as_searchable_or_filterable() { - let index = TempIndex::new(); - - index - .add_documents(documents!([ - { "id": 0, "_vectors": { "doggo": [2345] } }, - { "id": 1, "_vectors": { "doggo": [6789] } }, - ])) - .unwrap(); - - db_snap!(index, fields_ids_map, @r###" - 0 id | - 1 _vectors | - "###); - db_snap!(index, searchable_fields, @r###"["id"]"###); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - 0 0 | - "###); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let results = search.query("2345").execute().unwrap(); - assert!(results.candidates.is_empty()); - drop(rtxn); - - index - .update_settings(|settings| { - settings.set_searchable_fields(vec![S("_vectors"), S("_vectors.doggo")]); - settings.set_filterable_fields(vec![ - FilterableAttributesRule::Field("_vectors".to_string()), - FilterableAttributesRule::Field("_vectors.doggo".to_string()), - ]); - }) - .unwrap(); - - db_snap!(index, fields_ids_map, @r###" - 0 id | - 1 _vectors | - "###); - db_snap!(index, searchable_fields, @"[]"); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - "###); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let results = search.query("2345").execute().unwrap(); - assert!(results.candidates.is_empty()); - - let mut search = index.search(&rtxn); - let results = search - .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) - .execute() - .unwrap(); - assert!(results.candidates.is_empty()); - - index - .update_settings(|settings| { - settings.set_embedder_settings(btreemap! { - S("doggo") => Setting::Set(EmbeddingSettings { - dimensions: Setting::Set(1), - source: Setting::Set(EmbedderSource::UserProvided), - ..EmbeddingSettings::default()}), - }); - }) - .unwrap(); - - db_snap!(index, fields_ids_map, @r###" - 0 id | - 1 _vectors | - "###); - db_snap!(index, searchable_fields, @"[]"); - db_snap!(index, fieldids_weights_map, @r###" - fid weight - "###); - - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - let results = search.query("2345").execute().unwrap(); - assert!(results.candidates.is_empty()); - - let mut search = index.search(&rtxn); - let results = search - .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) - .execute() - .unwrap(); - assert!(results.candidates.is_empty()); - } -} +#[path = "test_index.rs"] +pub(crate) mod tests; diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index 516e6d31b..47d3dc75c 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -12,6 +12,7 @@ mod asc_desc; mod attribute_patterns; mod criterion; pub mod database_stats; +pub mod disabled_typos_terms; mod error; mod external_documents_ids; pub mod facet; diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 3505f7d4a..c3eba8031 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -1,10 +1,11 @@ use std::collections::BTreeSet; use std::fmt::{Debug, Display}; -use std::ops::Bound::{self, Excluded, Included}; +use std::ops::Bound::{self, Excluded, Included, Unbounded}; use either::Either; pub use filter_parser::{Condition, Error as FPError, FilterCondition, Token}; use heed::types::LazyDecode; +use heed::BytesEncode; use memchr::memmem::Finder; use roaring::{MultiOps, RoaringBitmap}; use serde_json::Value; @@ -14,7 +15,7 @@ use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::heed_codec::facet::{ - FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, OrderedF64Codec, + FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, }; use crate::index::db_name::FACET_ID_STRING_DOCIDS; use crate::{ @@ -271,7 +272,7 @@ impl<'a> Filter<'a> { // as the facets values are all in the same database and prefixed by the // field id and the level. - let (left, right) = match operator { + let (number_bounds, (left_str, right_str)) = match operator { // return an error if the filter is not allowed for this field Condition::GreaterThan(_) | Condition::GreaterThanOrEqual(_) @@ -305,17 +306,37 @@ impl<'a> Filter<'a> { )); } Condition::GreaterThan(val) => { - (Excluded(val.parse_finite_float()?), Included(f64::MAX)) + let number = val.parse_finite_float().ok(); + let number_bounds = number.map(|number| (Excluded(number), Included(f64::MAX))); + let str_bounds = (Excluded(val.value()), Unbounded); + (number_bounds, str_bounds) } Condition::GreaterThanOrEqual(val) => { - (Included(val.parse_finite_float()?), Included(f64::MAX)) + let number = val.parse_finite_float().ok(); + let number_bounds = number.map(|number| (Included(number), Included(f64::MAX))); + let str_bounds = (Included(val.value()), Unbounded); + (number_bounds, str_bounds) + } + Condition::LowerThan(val) => { + let number = val.parse_finite_float().ok(); + let number_bounds = number.map(|number| (Included(f64::MIN), Excluded(number))); + let str_bounds = (Unbounded, Excluded(val.value())); + (number_bounds, str_bounds) } - Condition::LowerThan(val) => (Included(f64::MIN), Excluded(val.parse_finite_float()?)), Condition::LowerThanOrEqual(val) => { - (Included(f64::MIN), Included(val.parse_finite_float()?)) + let number = val.parse_finite_float().ok(); + let number_bounds = number.map(|number| (Included(f64::MIN), Included(number))); + let str_bounds = (Unbounded, Included(val.value())); + (number_bounds, str_bounds) } Condition::Between { from, to } => { - (Included(from.parse_finite_float()?), Included(to.parse_finite_float()?)) + let from_number = from.parse_finite_float().ok(); + let to_number = to.parse_finite_float().ok(); + + let number_bounds = + from_number.zip(to_number).map(|(from, to)| (Included(from), Included(to))); + let str_bounds = (Included(from.value()), Included(to.value())); + (number_bounds, str_bounds) } Condition::Null => { let is_null = index.null_faceted_documents_ids(rtxn, field_id)?; @@ -415,29 +436,47 @@ impl<'a> Filter<'a> { }; let mut output = RoaringBitmap::new(); - Self::explore_facet_number_levels( + + if let Some((left_number, right_number)) = number_bounds { + Self::explore_facet_levels( + rtxn, + numbers_db, + field_id, + &left_number, + &right_number, + universe, + &mut output, + )?; + } + + Self::explore_facet_levels( rtxn, - numbers_db, + strings_db, field_id, - left, - right, + &left_str, + &right_str, universe, &mut output, )?; + Ok(output) } /// Aggregates the documents ids that are part of the specified range automatically /// going deeper through the levels. - fn explore_facet_number_levels( - rtxn: &heed::RoTxn<'_>, - db: heed::Database, FacetGroupValueCodec>, + fn explore_facet_levels<'data, BoundCodec>( + rtxn: &'data heed::RoTxn<'data>, + db: heed::Database, FacetGroupValueCodec>, field_id: FieldId, - left: Bound, - right: Bound, + left: &'data Bound<>::EItem>, + right: &'data Bound<>::EItem>, universe: Option<&RoaringBitmap>, output: &mut RoaringBitmap, - ) -> Result<()> { + ) -> Result<()> + where + BoundCodec: for<'b> BytesEncode<'b>, + for<'b> >::EItem: Sized + PartialOrd, + { match (left, right) { // lower TO upper when lower > upper must return no result (Included(l), Included(r)) if l > r => return Ok(()), @@ -446,8 +485,8 @@ impl<'a> Filter<'a> { (Excluded(l), Included(r)) if l >= r => return Ok(()), (_, _) => (), } - facet_range_search::find_docids_of_facet_within_bounds::( - rtxn, db, field_id, &left, &right, universe, output, + facet_range_search::find_docids_of_facet_within_bounds::( + rtxn, db, field_id, left, right, universe, output, )?; Ok(()) @@ -1249,28 +1288,24 @@ mod tests { let result = filter.evaluate(&rtxn, &index).unwrap(); assert!(result.contains(0)); let filter = Filter::from_str("price < inf").unwrap().unwrap(); - assert!(matches!( - filter.evaluate(&rtxn, &index), - Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_))) - )); + let result = filter.evaluate(&rtxn, &index).unwrap(); + // this is allowed due to filters with strings + assert!(result.contains(1)); let filter = Filter::from_str("price = NaN").unwrap().unwrap(); let result = filter.evaluate(&rtxn, &index).unwrap(); assert!(result.is_empty()); let filter = Filter::from_str("price < NaN").unwrap().unwrap(); - assert!(matches!( - filter.evaluate(&rtxn, &index), - Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_))) - )); + let result = filter.evaluate(&rtxn, &index).unwrap(); + assert!(result.contains(1)); let filter = Filter::from_str("price = infinity").unwrap().unwrap(); let result = filter.evaluate(&rtxn, &index).unwrap(); assert!(result.contains(2)); let filter = Filter::from_str("price < infinity").unwrap().unwrap(); - assert!(matches!( - filter.evaluate(&rtxn, &index), - Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_))) - )); + let result = filter.evaluate(&rtxn, &index).unwrap(); + assert!(result.contains(0)); + assert!(result.contains(1)); } #[test] diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index def00ec92..37b1aaf09 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -8,7 +8,7 @@ use roaring::bitmap::RoaringBitmap; pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET}; pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords}; -use self::new::{execute_vector_search, PartialSearchResult}; +use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::vector::Embedder; @@ -269,6 +269,12 @@ impl<'a> Search<'a> { )?, }; + if let Some(VectorStoreStats { total_time, total_queries, total_results }) = + ctx.vector_store_stats + { + tracing::debug!("Vector store stats: total_time={total_time:.02?}, total_queries={total_queries}, total_results={total_results}"); + } + // consume context and located_query_terms to build MatchingWords. let matching_words = match located_query_terms { Some(located_query_terms) => MatchingWords::new(ctx, located_query_terms), diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index 8b04d8c6a..6e794ef53 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -22,6 +22,8 @@ mod vector_sort; mod tests; use std::collections::HashSet; +use std::ops::AddAssign; +use std::time::Duration; use bucket_sort::{bucket_sort, BucketSortOutput}; use charabia::{Language, TokenizerBuilder}; @@ -72,6 +74,7 @@ pub struct SearchContext<'ctx> { pub phrase_docids: PhraseDocIdsCache, pub restricted_fids: Option, pub prefix_search: PrefixSearch, + pub vector_store_stats: Option, } impl<'ctx> SearchContext<'ctx> { @@ -101,6 +104,7 @@ impl<'ctx> SearchContext<'ctx> { phrase_docids: <_>::default(), restricted_fids: None, prefix_search, + vector_store_stats: None, }) } @@ -166,6 +170,25 @@ impl<'ctx> SearchContext<'ctx> { } } +#[derive(Debug, Default)] +pub struct VectorStoreStats { + /// The total time spent on vector search. + pub total_time: Duration, + /// The number of searches performed. + pub total_queries: usize, + /// The number of nearest neighbors found. + pub total_results: usize, +} + +impl AddAssign for VectorStoreStats { + fn add_assign(&mut self, other: Self) { + let Self { total_time, total_queries, total_results } = self; + *total_time += other.total_time; + *total_queries += other.total_queries; + *total_results += other.total_results; + } +} + #[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)] pub enum Word { Original(Interned), diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index a25605cfc..834f97384 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -1,8 +1,10 @@ use std::iter::FromIterator; +use std::time::Instant; use roaring::RoaringBitmap; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; +use super::VectorStoreStats; use crate::score_details::{self, ScoreDetails}; use crate::vector::{ArroyWrapper, DistributionShift, Embedder}; use crate::{DocumentId, Result, SearchContext, SearchLogger}; @@ -53,9 +55,15 @@ impl VectorSort { ) -> Result<()> { let target = &self.target; + let before = Instant::now(); let reader = ArroyWrapper::new(ctx.index.vector_arroy, self.embedder_index, self.quantized); let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; self.cached_sorted_docids = results.into_iter(); + *ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats { + total_time: before.elapsed(), + total_queries: 1, + total_results: self.cached_sorted_docids.len(), + }; Ok(()) } diff --git a/crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap deleted file mode 100644 index f9ebc0c20..000000000 --- a/crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap +++ /dev/null @@ -1,4 +0,0 @@ ---- -source: milli/src/index.rs ---- -[0, ] diff --git a/crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap deleted file mode 100644 index 89fb1856a..000000000 --- a/crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap +++ /dev/null @@ -1,4 +0,0 @@ ---- -source: milli/src/index.rs ---- -[] diff --git a/crates/milli/src/snapshots/test_index.rs/bug_3007/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/test_index.rs/bug_3007/geo_faceted_documents_ids.snap new file mode 100644 index 000000000..28f66783a --- /dev/null +++ b/crates/milli/src/snapshots/test_index.rs/bug_3007/geo_faceted_documents_ids.snap @@ -0,0 +1,4 @@ +--- +source: crates/milli/src/test_index.rs +--- +[0, ] diff --git a/crates/milli/src/snapshots/test_index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap b/crates/milli/src/snapshots/test_index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap new file mode 100644 index 000000000..72a8cceb6 --- /dev/null +++ b/crates/milli/src/snapshots/test_index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap @@ -0,0 +1,4 @@ +--- +source: crates/milli/src/test_index.rs +--- +[] diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs new file mode 100644 index 000000000..7759b3e18 --- /dev/null +++ b/crates/milli/src/test_index.rs @@ -0,0 +1,1399 @@ +use std::collections::HashSet; +use std::ops::Deref; + +use big_s::S; +use bumpalo::Bump; +use heed::{EnvOpenOptions, RwTxn}; +use maplit::btreemap; +use memmap2::Mmap; +use tempfile::TempDir; + +use crate::constants::RESERVED_GEO_FIELD_NAME; +use crate::error::{Error, InternalError}; +use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; +use crate::progress::Progress; +use crate::update::new::indexer; +use crate::update::settings::InnerIndexSettings; +use crate::update::{ + self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings, +}; +use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; +use crate::vector::EmbeddingConfigs; +use crate::{ + db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, + ThreadPoolNoAbortBuilder, +}; + +pub(crate) struct TempIndex { + pub inner: Index, + pub indexer_config: IndexerConfig, + pub index_documents_config: IndexDocumentsConfig, + _tempdir: TempDir, +} + +impl Deref for TempIndex { + type Target = Index; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl TempIndex { + /// Creates a temporary index + pub fn new_with_map_size(size: usize) -> Self { + let options = EnvOpenOptions::new(); + let mut options = options.read_txn_without_tls(); + options.map_size(size); + let _tempdir = TempDir::new_in(".").unwrap(); + let inner = Index::new(options, _tempdir.path(), true).unwrap(); + let indexer_config = IndexerConfig::default(); + let index_documents_config = IndexDocumentsConfig::default(); + Self { inner, indexer_config, index_documents_config, _tempdir } + } + /// Creates a temporary index, with a default `4096 * 2000` size. This should be enough for + /// most tests. + pub fn new() -> Self { + Self::new_with_map_size(4096 * 2000) + } + + pub fn add_documents_using_wtxn<'t>( + &'t self, + wtxn: &mut RwTxn<'t>, + documents: Mmap, + ) -> Result<(), crate::error::Error> { + let local_pool; + let indexer_config = &self.indexer_config; + let pool = match &indexer_config.thread_pool { + Some(pool) => pool, + None => { + local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap(); + &local_pool + } + }; + + let rtxn = self.inner.read_txn()?; + let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?; + let mut new_fields_ids_map = db_fields_ids_map.clone(); + + let embedders = InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs; + let mut indexer = indexer::DocumentOperation::new(); + match self.index_documents_config.update_method { + IndexDocumentsMethod::ReplaceDocuments => { + indexer.replace_documents(&documents).unwrap() + } + IndexDocumentsMethod::UpdateDocuments => indexer.update_documents(&documents).unwrap(), + } + + let indexer_alloc = Bump::new(); + let (document_changes, operation_stats, primary_key) = indexer.into_changes( + &indexer_alloc, + &self.inner, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + Progress::default(), + )?; + + if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { + return Err(error.into()); + } + + pool.install(|| { + indexer::index( + wtxn, + &self.inner, + &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), + indexer_config.grenad_parameters(), + &db_fields_ids_map, + new_fields_ids_map, + primary_key, + &document_changes, + embedders, + &|| false, + &Progress::default(), + ) + }) + .unwrap()?; + + Ok(()) + } + + pub fn add_documents(&self, documents: Mmap) -> Result<(), crate::error::Error> { + let mut wtxn = self.write_txn().unwrap(); + self.add_documents_using_wtxn(&mut wtxn, documents)?; + wtxn.commit().unwrap(); + Ok(()) + } + + pub fn update_settings( + &self, + update: impl Fn(&mut Settings<'_, '_, '_>), + ) -> Result<(), crate::error::Error> { + let mut wtxn = self.write_txn().unwrap(); + self.update_settings_using_wtxn(&mut wtxn, update)?; + wtxn.commit().unwrap(); + Ok(()) + } + + pub fn update_settings_using_wtxn<'t>( + &'t self, + wtxn: &mut RwTxn<'t>, + update: impl Fn(&mut Settings<'_, '_, '_>), + ) -> Result<(), crate::error::Error> { + let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config); + update(&mut builder); + builder.execute(drop, || false)?; + Ok(()) + } + + pub fn delete_documents_using_wtxn<'t>( + &'t self, + wtxn: &mut RwTxn<'t>, + external_document_ids: Vec, + ) -> Result<(), crate::error::Error> { + let local_pool; + let indexer_config = &self.indexer_config; + let pool = match &indexer_config.thread_pool { + Some(pool) => pool, + None => { + local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap(); + &local_pool + } + }; + + let rtxn = self.inner.read_txn()?; + let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?; + let mut new_fields_ids_map = db_fields_ids_map.clone(); + + let embedders = InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs; + + let mut indexer = indexer::DocumentOperation::new(); + let external_document_ids: Vec<_> = + external_document_ids.iter().map(AsRef::as_ref).collect(); + indexer.delete_documents(external_document_ids.as_slice()); + + let indexer_alloc = Bump::new(); + let (document_changes, operation_stats, primary_key) = indexer.into_changes( + &indexer_alloc, + &self.inner, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + Progress::default(), + )?; + + if let Some(error) = operation_stats.into_iter().find_map(|stat| stat.error) { + return Err(error.into()); + } + + pool.install(|| { + indexer::index( + wtxn, + &self.inner, + &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), + indexer_config.grenad_parameters(), + &db_fields_ids_map, + new_fields_ids_map, + primary_key, + &document_changes, + embedders, + &|| false, + &Progress::default(), + ) + }) + .unwrap()?; + + Ok(()) + } + + pub fn delete_documents(&self, external_document_ids: Vec) { + let mut wtxn = self.write_txn().unwrap(); + + self.delete_documents_using_wtxn(&mut wtxn, external_document_ids).unwrap(); + + wtxn.commit().unwrap(); + } + + pub fn delete_document(&self, external_document_id: &str) { + self.delete_documents(vec![external_document_id.to_string()]) + } +} + +#[test] +fn aborting_indexation() { + use std::sync::atomic::AtomicBool; + use std::sync::atomic::Ordering::Relaxed; + + let index = TempIndex::new(); + let mut wtxn = index.inner.write_txn().unwrap(); + let should_abort = AtomicBool::new(false); + + let local_pool; + let indexer_config = &index.indexer_config; + let pool = match &indexer_config.thread_pool { + Some(pool) => pool, + None => { + local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap(); + &local_pool + } + }; + + let rtxn = index.inner.read_txn().unwrap(); + let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap(); + let mut new_fields_ids_map = db_fields_ids_map.clone(); + + let embedders = EmbeddingConfigs::default(); + let mut indexer = indexer::DocumentOperation::new(); + let payload = documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "bob", "age": 20 }, + { "id": 2, "name": "bob", "age": 20 }, + ]); + indexer.replace_documents(&payload).unwrap(); + + let indexer_alloc = Bump::new(); + let (document_changes, _operation_stats, primary_key) = indexer + .into_changes( + &indexer_alloc, + &index.inner, + &rtxn, + None, + &mut new_fields_ids_map, + &|| false, + Progress::default(), + ) + .unwrap(); + + should_abort.store(true, Relaxed); + + let err = pool + .install(|| { + indexer::index( + &mut wtxn, + &index.inner, + &crate::ThreadPoolNoAbortBuilder::new().build().unwrap(), + indexer_config.grenad_parameters(), + &db_fields_ids_map, + new_fields_ids_map, + primary_key, + &document_changes, + embedders, + &|| should_abort.load(Relaxed), + &Progress::default(), + ) + }) + .unwrap() + .unwrap_err(); + + assert!(matches!(err, Error::InternalError(InternalError::AbortedIndexation))); +} + +#[test] +fn initial_field_distribution() { + let index = TempIndex::new(); + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "bob", "age": 20 }, + { "id": 2, "name": "bob", "age": 20 }, + ])) + .unwrap(); + + db_snap!(index, field_distribution, @r###" + age 1 | + id 2 | + name 2 | + "###); + + db_snap!(index, word_docids, + @r###" + 1 [0, ] + 2 [1, ] + 20 [1, ] + bob [1, ] + kevin [0, ] + "### + ); + + // we add all the documents a second time. we are supposed to get the same + // field_distribution in the end + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "bob", "age": 20 }, + { "id": 2, "name": "bob", "age": 20 }, + ])) + .unwrap(); + + db_snap!(index, field_distribution, + @r###" + age 1 | + id 2 | + name 2 | + "### + ); + + // then we update a document by removing one field and another by adding one field + index + .add_documents(documents!([ + { "id": 1, "name": "kevin", "has_dog": true }, + { "id": 2, "name": "bob" } + ])) + .unwrap(); + + db_snap!(index, field_distribution, + @r###" + has_dog 1 | + id 2 | + name 2 | + "### + ); +} + +#[test] +fn put_and_retrieve_disable_typo() { + let index = TempIndex::new(); + let mut txn = index.write_txn().unwrap(); + // default value is true + assert!(index.authorize_typos(&txn).unwrap()); + // set to false + index.put_authorize_typos(&mut txn, false).unwrap(); + txn.commit().unwrap(); + + let txn = index.read_txn().unwrap(); + assert!(!index.authorize_typos(&txn).unwrap()); +} + +#[test] +fn set_min_word_len_for_typos() { + let index = TempIndex::new(); + let mut txn = index.write_txn().unwrap(); + + assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_ONE_TYPO); + assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_TWO_TYPOS); + + index.put_min_word_len_one_typo(&mut txn, 3).unwrap(); + index.put_min_word_len_two_typos(&mut txn, 15).unwrap(); + + txn.commit().unwrap(); + + let txn = index.read_txn().unwrap(); + assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), 3); + assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), 15); +} + +#[test] +fn add_documents_and_set_searchable_fields() { + let index = TempIndex::new(); + index + .add_documents(documents!([ + { "id": 1, "doggo": "kevin" }, + { "id": 2, "doggo": { "name": "bob", "age": 20 } }, + { "id": 3, "name": "jean", "age": 25 }, + ])) + .unwrap(); + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("doggo"), S("name")]); + }) + .unwrap(); + + // ensure we get the right real searchable fields + user defined searchable fields + let rtxn = index.read_txn().unwrap(); + + let real = index.searchable_fields(&rtxn).unwrap(); + assert_eq!(real, &["doggo", "name", "doggo.name", "doggo.age"]); + + let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); + assert_eq!(user_defined, &["doggo", "name"]); +} + +#[test] +fn set_searchable_fields_and_add_documents() { + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("doggo"), S("name")]); + }) + .unwrap(); + + // ensure we get the right real searchable fields + user defined searchable fields + let rtxn = index.read_txn().unwrap(); + + let real = index.searchable_fields(&rtxn).unwrap(); + assert!(real.is_empty()); + let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); + assert_eq!(user_defined, &["doggo", "name"]); + + index + .add_documents(documents!([ + { "id": 1, "doggo": "kevin" }, + { "id": 2, "doggo": { "name": "bob", "age": 20 } }, + { "id": 3, "name": "jean", "age": 25 }, + ])) + .unwrap(); + + // ensure we get the right real searchable fields + user defined searchable fields + let rtxn = index.read_txn().unwrap(); + + let real = index.searchable_fields(&rtxn).unwrap(); + assert_eq!(real, &["doggo", "name", "doggo.name", "doggo.age"]); + + let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); + assert_eq!(user_defined, &["doggo", "name"]); +} + +#[test] +fn test_basic_geo_bounding_box() { + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_filterable_fields(vec![FilterableAttributesRule::Field( + RESERVED_GEO_FIELD_NAME.to_string(), + )]); + }) + .unwrap(); + index + .add_documents(documents!([ + { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": "0" } }, + { "id": 1, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": "-175" } }, + { "id": 2, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": 175 } }, + { "id": 3, RESERVED_GEO_FIELD_NAME: { "lat": 85, "lng": 0 } }, + { "id": 4, RESERVED_GEO_FIELD_NAME: { "lat": "-85", "lng": "0" } }, + ])) + .unwrap(); + + // ensure we get the right real searchable fields + user defined searchable fields + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + + // exact match a document + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([0, 0], [0, 0])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0]>"); + + // match a document in the middle of the rectangle + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([10, 10], [-10, -10])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0]>"); + + // select everything + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([90, 180], [-90, -180])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1, 2, 3, 4]>"); + + // go on the edge of the longitude + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([0, -170], [0, 180])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[1]>"); + + // go on the other edge of the longitude + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([0, -180], [0, 170])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[2]>"); + + // wrap around the longitude + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([0, -170], [0, 170])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[1, 2]>"); + + // go on the edge of the latitude + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([90, 0], [80, 0])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[3]>"); + + // go on the edge of the latitude + let search_result = search + .filter(Filter::from_str("_geoBoundingBox([-80, 0], [-90, 0])").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[4]>"); + + // the requests that don't make sense + + // try to wrap around the latitude + let error = search + .filter(Filter::from_str("_geoBoundingBox([-80, 0], [80, 0])").unwrap().unwrap()) + .execute() + .unwrap_err(); + insta::assert_snapshot!( + error, + @r###" + The top latitude `-80` is below the bottom latitude `80`. + 32:33 _geoBoundingBox([-80, 0], [80, 0]) + "### + ); + + // send a top latitude lower than the bottow latitude + let error = search + .filter(Filter::from_str("_geoBoundingBox([-10, 0], [10, 0])").unwrap().unwrap()) + .execute() + .unwrap_err(); + insta::assert_snapshot!( + error, + @r###" + The top latitude `-10` is below the bottom latitude `10`. + 32:33 _geoBoundingBox([-10, 0], [10, 0]) + "### + ); +} + +#[test] +fn test_contains() { + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings + .set_filterable_fields(vec![FilterableAttributesRule::Field("doggo".to_string())]); + }) + .unwrap(); + index + .add_documents(documents!([ + { "id": 0, "doggo": "kefir" }, + { "id": 1, "doggo": "kefirounet" }, + { "id": 2, "doggo": "kefkef" }, + { "id": 3, "doggo": "fifir" }, + { "id": 4, "doggo": "boubou" }, + { "id": 5 }, + ])) + .unwrap(); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let search_result = search + .filter(Filter::from_str("doggo CONTAINS kefir").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1]>"); + let mut search = index.search(&rtxn); + let search_result = + search.filter(Filter::from_str("doggo CONTAINS KEF").unwrap().unwrap()).execute().unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[0, 1, 2]>"); + let mut search = index.search(&rtxn); + let search_result = search + .filter(Filter::from_str("doggo NOT CONTAINS fir").unwrap().unwrap()) + .execute() + .unwrap(); + insta::assert_debug_snapshot!(search_result.candidates, @"RoaringBitmap<[2, 4, 5]>"); +} + +#[test] +fn replace_documents_external_ids_and_soft_deletion_check() { + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_primary_key("id".to_owned()); + settings + .set_filterable_fields(vec![FilterableAttributesRule::Field("doggo".to_string())]); + }) + .unwrap(); + + let mut docs = vec![]; + for i in 0..4 { + docs.push(serde_json::json!( + { "id": i, "doggo": i } + )); + } + index.add_documents(documents!(docs)).unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); + db_snap!(index, external_documents_ids, 1, @r###" + docids: + 0 0 + 1 1 + 2 2 + 3 3 + "###); + db_snap!(index, facet_id_f64_docids, 1, @r###" + 1 0 0 1 [0, ] + 1 0 1 1 [1, ] + 1 0 2 1 [2, ] + 1 0 3 1 [3, ] + "###); + + let mut docs = vec![]; + for i in 0..3 { + docs.push(serde_json::json!( + { "id": i, "doggo": i + 1 } + )); + } + index.add_documents(documents!(docs)).unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 0 0 + 1 1 + 2 2 + 3 3 + "###); + db_snap!(index, facet_id_f64_docids, 2, @r###" + 1 0 1 1 [0, ] + 1 0 2 1 [1, ] + 1 0 3 1 [2, 3, ] + "###); + + index + .add_documents( + documents!([{ "id": 3, "doggo": 4 }, { "id": 3, "doggo": 5 },{ "id": 3, "doggo": 4 }]), + ) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); + db_snap!(index, external_documents_ids, 3, @r###" + docids: + 0 0 + 1 1 + 2 2 + 3 3 + "###); + db_snap!(index, facet_id_f64_docids, 3, @r###" + 1 0 1 1 [0, ] + 1 0 2 1 [1, ] + 1 0 3 1 [2, ] + 1 0 4 1 [3, ] + "###); + + index + .update_settings(|settings| { + settings.set_distinct_field("id".to_owned()); + }) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, ]"); + db_snap!(index, external_documents_ids, 3, @r###" + docids: + 0 0 + 1 1 + 2 2 + 3 3 + "###); + db_snap!(index, facet_id_f64_docids, 3, @r###" + 0 0 0 1 [0, ] + 0 0 1 1 [1, ] + 0 0 2 1 [2, ] + 0 0 3 1 [3, ] + 1 0 1 1 [0, ] + 1 0 2 1 [1, ] + 1 0 3 1 [2, ] + 1 0 4 1 [3, ] + "###); +} + +#[test] +fn bug_3021_first() { + // https://github.com/meilisearch/meilisearch/issues/3021 + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; + + index + .update_settings(|settings| { + settings.set_primary_key("primary_key".to_owned()); + }) + .unwrap(); + + index + .add_documents(documents!([ + { "primary_key": 38 }, + { "primary_key": 34 } + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + db_snap!(index, external_documents_ids, 1, @r###" + docids: + 34 1 + 38 0 + "###); + + index.delete_document("34"); + + db_snap!(index, documents_ids, @"[0, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 38 0 + "###); + + index + .update_settings(|s| { + s.set_searchable_fields(vec![]); + }) + .unwrap(); + + // The key point of the test is to verify that the external documents ids + // do not contain any entry for previously soft-deleted document ids + db_snap!(index, documents_ids, @"[0, ]"); + db_snap!(index, external_documents_ids, 3, @r###" + docids: + 38 0 + "###); + + // So that this document addition works correctly now. + // It would be wrongly interpreted as a replacement before + index.add_documents(documents!({ "primary_key": 34 })).unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + db_snap!(index, external_documents_ids, 4, @r###" + docids: + 34 1 + 38 0 + "###); + + // We do the test again, but deleting the document with id 0 instead of id 1 now + index.delete_document("38"); + + db_snap!(index, documents_ids, @"[1, ]"); + db_snap!(index, external_documents_ids, 5, @r###" + docids: + 34 1 + "###); + + index + .update_settings(|s| { + s.set_searchable_fields(vec!["primary_key".to_owned()]); + }) + .unwrap(); + + db_snap!(index, documents_ids, @"[1, ]"); + db_snap!(index, external_documents_ids, 6, @r###" + docids: + 34 1 + "###); + + // And adding lots of documents afterwards instead of just one. + // These extra subtests don't add much, but it's better than nothing. + index + .add_documents(documents!([ + { "primary_key": 38 }, + { "primary_key": 39 }, + { "primary_key": 41 }, + { "primary_key": 40 }, + { "primary_key": 41 }, + { "primary_key": 42 }, + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, 4, 5, ]"); + db_snap!(index, external_documents_ids, 7, @r###" + docids: + 34 1 + 38 0 + 39 2 + 40 4 + 41 3 + 42 5 + "###); +} + +#[test] +fn simple_delete() { + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; + index + .add_documents(documents!([ + { "id": 30 }, + { "id": 34 } + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + db_snap!(index, external_documents_ids, 1, @r###" + docids: + 30 0 + 34 1"###); + + index.delete_document("34"); + + db_snap!(index, documents_ids, @"[0, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 30 0 + "###); +} + +#[test] +fn bug_3021_second() { + // https://github.com/meilisearch/meilisearch/issues/3021 + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; + + index + .update_settings(|settings| { + settings.set_primary_key("primary_key".to_owned()); + }) + .unwrap(); + + index + .add_documents(documents!([ + { "primary_key": 30 }, + { "primary_key": 34 } + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + db_snap!(index, external_documents_ids, 1, @r###" + docids: + 30 0 + 34 1 + "###); + + index.delete_document("34"); + + db_snap!(index, documents_ids, @"[0, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 30 0 + "###); + + index + .update_settings(|s| { + s.set_searchable_fields(vec![]); + }) + .unwrap(); + + // The key point of the test is to verify that the external documents ids + // do not contain any entry for previously soft-deleted document ids + db_snap!(index, documents_ids, @"[0, ]"); + db_snap!(index, external_documents_ids, 3, @r###" + docids: + 30 0 + "###); + + // So that when we add a new document + index.add_documents(documents!({ "primary_key": 35, "b": 2 })).unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + // The external documents ids don't have several external ids pointing to the same + // internal document id + db_snap!(index, external_documents_ids, 4, @r###" + docids: + 30 0 + 35 1 + "###); + + // And when we add 34 again, we don't replace document 35 + index.add_documents(documents!({ "primary_key": 34, "a": 1 })).unwrap(); + + // And document 35 still exists, is not deleted + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, 5, @r###" + docids: + 30 0 + 34 2 + 35 1 + "###); + + let rtxn = index.read_txn().unwrap(); + let (_docid, obkv) = index.documents(&rtxn, [0]).unwrap()[0]; + let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); + insta::assert_debug_snapshot!(json, @r###" + { + "primary_key": Number(30), + } + "###); + + // Furthermore, when we retrieve document 34, it is not the result of merging 35 with 34 + let (_docid, obkv) = index.documents(&rtxn, [2]).unwrap()[0]; + let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); + insta::assert_debug_snapshot!(json, @r###" + { + "primary_key": Number(34), + "a": Number(1), + } + "###); + + drop(rtxn); + + // Add new documents again + index + .add_documents( + documents!([{ "primary_key": 37 }, { "primary_key": 38 }, { "primary_key": 39 }]), + ) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, 3, 4, 5, ]"); + db_snap!(index, external_documents_ids, 6, @r###" + docids: + 30 0 + 34 2 + 35 1 + 37 3 + 38 4 + 39 5 + "###); +} + +#[test] +fn bug_3021_third() { + // https://github.com/meilisearch/meilisearch/issues/3021 + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; + + index + .update_settings(|settings| { + settings.set_primary_key("primary_key".to_owned()); + }) + .unwrap(); + + index + .add_documents(documents!([ + { "primary_key": 3 }, + { "primary_key": 4 }, + { "primary_key": 5 } + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, 1, @r###" + docids: + 3 0 + 4 1 + 5 2 + "###); + + index.delete_document("3"); + + db_snap!(index, documents_ids, @"[1, 2, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 4 1 + 5 2 + "###); + + index.add_documents(documents!([{ "primary_key": "4", "a": 2 }])).unwrap(); + + db_snap!(index, documents_ids, @"[1, 2, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 4 1 + 5 2 + "###); + + index + .add_documents(documents!([ + { "primary_key": "3" }, + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, 2, @r###" + docids: + 3 0 + 4 1 + 5 2 + "###); +} + +#[test] +fn bug_3021_fourth() { + // https://github.com/meilisearch/meilisearch/issues/3021 + let mut index = TempIndex::new(); + index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; + + index + .update_settings(|settings| { + settings.set_primary_key("primary_key".to_owned()); + }) + .unwrap(); + + index + .add_documents(documents!([ + { "primary_key": 11 }, + { "primary_key": 4 }, + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, ]"); + db_snap!(index, external_documents_ids, @r###" + docids: + 11 0 + 4 1 + "###); + db_snap!(index, fields_ids_map, @r###" + 0 primary_key | + "###); + db_snap!(index, searchable_fields, @r###"["primary_key"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 0 0 | + "###); + + index + .add_documents(documents!([ + { "primary_key": 4, "a": 0 }, + { "primary_key": 1 }, + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, @r###" + docids: + 1 2 + 11 0 + 4 1 + "###); + db_snap!(index, fields_ids_map, @r###" + 0 primary_key | + 1 a | + "###); + db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 0 0 | + 1 0 | + "###); + + index.delete_documents(Default::default()); + + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, @r###" + docids: + 1 2 + 11 0 + 4 1 + "###); + db_snap!(index, fields_ids_map, @r###" + 0 primary_key | + 1 a | + "###); + db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 0 0 | + 1 0 | + "###); + + index + .add_documents(documents!([ + { "primary_key": 4, "a": 1 }, + { "primary_key": 1, "a": 0 }, + ])) + .unwrap(); + + db_snap!(index, documents_ids, @"[0, 1, 2, ]"); + db_snap!(index, external_documents_ids, @r###" + docids: + 1 2 + 11 0 + 4 1 + "###); + db_snap!(index, fields_ids_map, @r###" + 0 primary_key | + 1 a | + "###); + db_snap!(index, searchable_fields, @r###"["primary_key", "a"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 0 0 | + 1 0 | + "###); + + let rtxn = index.read_txn().unwrap(); + let search = Search::new(&rtxn, &index); + let SearchResult { + matching_words: _, + candidates: _, + document_scores: _, + mut documents_ids, + degraded: _, + used_negative_operator: _, + } = search.execute().unwrap(); + let primary_key_id = index.fields_ids_map(&rtxn).unwrap().id("primary_key").unwrap(); + documents_ids.sort_unstable(); + let docs = index.documents(&rtxn, documents_ids).unwrap(); + let mut all_ids = HashSet::new(); + for (_docid, obkv) in docs { + let id = obkv.get(primary_key_id).unwrap(); + assert!(all_ids.insert(id)); + } +} + +#[test] +fn bug_3007() { + // https://github.com/meilisearch/meilisearch/issues/3007 + + use crate::error::{GeoError, UserError}; + let index = TempIndex::new(); + + // Given is an index with a geo field NOT contained in the sortable_fields of the settings + index + .update_settings(|settings| { + settings.set_primary_key("id".to_string()); + settings.set_filterable_fields(vec![FilterableAttributesRule::Field( + RESERVED_GEO_FIELD_NAME.to_string(), + )]); + }) + .unwrap(); + + // happy path + index + .add_documents(documents!({ "id" : 5, RESERVED_GEO_FIELD_NAME: {"lat": 12.0, "lng": 11.0}})) + .unwrap(); + + db_snap!(index, geo_faceted_documents_ids); + + // both are unparseable, we expect GeoError::BadLatitudeAndLongitude + let err1 = index + .add_documents( + documents!({ "id" : 6, RESERVED_GEO_FIELD_NAME: {"lat": "unparseable", "lng": "unparseable"}}), + ) + .unwrap_err(); + match err1 { + Error::UserError(UserError::InvalidGeoField(err)) => match *err { + GeoError::BadLatitudeAndLongitude { .. } => (), + otherwise => { + panic!("err1 is not a BadLatitudeAndLongitude error but rather a {otherwise:?}") + } + }, + _ => panic!("err1 is not a BadLatitudeAndLongitude error but rather a {err1:?}"), + } + + db_snap!(index, geo_faceted_documents_ids); // ensure that no more document was inserted +} + +#[test] +fn unexpected_extra_fields_in_geo_field() { + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_primary_key("id".to_string()); + settings.set_filterable_fields(vec![FilterableAttributesRule::Field( + RESERVED_GEO_FIELD_NAME.to_string(), + )]); + }) + .unwrap(); + + let err = index + .add_documents( + documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best" }}), + ) + .unwrap_err(); + insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"doggo":"are the best"}`."###); + + db_snap!(index, geo_faceted_documents_ids); // ensure that no documents were inserted + + // multiple fields and complex values + let err = index + .add_documents( + documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best", "and": { "all": ["cats", { "are": "beautiful" } ] } } }), + ) + .unwrap_err(); + insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"and":{"all":["cats",{"are":"beautiful"}]},"doggo":"are the best"}`."###); + + db_snap!(index, geo_faceted_documents_ids); // ensure that no documents were inserted +} + +#[test] +fn swapping_searchable_attributes() { + // See https://github.com/meilisearch/meilisearch/issues/4484 + + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("name")]); + settings + .set_filterable_fields(vec![FilterableAttributesRule::Field("age".to_string())]); + }) + .unwrap(); + + index + .add_documents(documents!({ "id": 1, "name": "Many", "age": 28, "realName": "Maxime" })) + .unwrap(); + db_snap!(index, fields_ids_map, @r###" + 0 id | + 1 name | + 2 age | + 3 realName | + "###); + db_snap!(index, searchable_fields, @r###"["name"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 1 0 | + "###); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("name"), S("realName")]); + settings + .set_filterable_fields(vec![FilterableAttributesRule::Field("age".to_string())]); + }) + .unwrap(); + + // The order of the field id map shouldn't change + db_snap!(index, fields_ids_map, @r###" + 0 id | + 1 name | + 2 age | + 3 realName | + "###); + db_snap!(index, searchable_fields, @r###"["name", "realName"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 1 0 | + 3 1 | + "###); +} + +#[test] +fn attribute_weights_after_swapping_searchable_attributes() { + // See https://github.com/meilisearch/meilisearch/issues/4484 + + let index = TempIndex::new(); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("name"), S("beverage")]); + }) + .unwrap(); + + index + .add_documents(documents!([ + { "id": 0, "name": "kefir", "beverage": "water" }, + { "id": 1, "name": "tamo", "beverage": "kefir" } + ])) + .unwrap(); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let results = search.query("kefir").execute().unwrap(); + + // We should find kefir the dog first + insta::assert_debug_snapshot!(results.documents_ids, @r###" + [ + 0, + 1, + ] + "###); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("beverage"), S("name")]); + }) + .unwrap(); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let results = search.query("kefir").execute().unwrap(); + + // We should find tamo first + insta::assert_debug_snapshot!(results.documents_ids, @r###" + [ + 1, + 0, + ] + "###); +} + +#[test] +fn vectors_are_never_indexed_as_searchable_or_filterable() { + let index = TempIndex::new(); + + index + .add_documents(documents!([ + { "id": 0, "_vectors": { "doggo": [2345] } }, + { "id": 1, "_vectors": { "doggo": [6789] } }, + ])) + .unwrap(); + + db_snap!(index, fields_ids_map, @r###" + 0 id | + 1 _vectors | + "###); + db_snap!(index, searchable_fields, @r###"["id"]"###); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + 0 0 | + "###); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let results = search.query("2345").execute().unwrap(); + assert!(results.candidates.is_empty()); + drop(rtxn); + + index + .update_settings(|settings| { + settings.set_searchable_fields(vec![S("_vectors"), S("_vectors.doggo")]); + settings.set_filterable_fields(vec![ + FilterableAttributesRule::Field("_vectors".to_string()), + FilterableAttributesRule::Field("_vectors.doggo".to_string()), + ]); + }) + .unwrap(); + + db_snap!(index, fields_ids_map, @r###" + 0 id | + 1 _vectors | + "###); + db_snap!(index, searchable_fields, @"[]"); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + "###); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let results = search.query("2345").execute().unwrap(); + assert!(results.candidates.is_empty()); + + let mut search = index.search(&rtxn); + let results = search + .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) + .execute() + .unwrap(); + assert!(results.candidates.is_empty()); + + index + .update_settings(|settings| { + settings.set_embedder_settings(btreemap! { + S("doggo") => Setting::Set(EmbeddingSettings { + dimensions: Setting::Set(1), + source: Setting::Set(EmbedderSource::UserProvided), + ..EmbeddingSettings::default()}), + }); + }) + .unwrap(); + + db_snap!(index, fields_ids_map, @r###" + 0 id | + 1 _vectors | + "###); + db_snap!(index, searchable_fields, @"[]"); + db_snap!(index, fieldids_weights_map, @r###" + fid weight + "###); + + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + let results = search.query("2345").execute().unwrap(); + assert!(results.candidates.is_empty()); + + let mut search = index.search(&rtxn); + let results = search + .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) + .execute() + .unwrap(); + assert!(results.candidates.is_empty()); +} diff --git a/crates/milli/src/update/index_documents/extract/extract_word_docids.rs b/crates/milli/src/update/index_documents/extract/extract_word_docids.rs index 829da768c..a964c0bbe 100644 --- a/crates/milli/src/update/index_documents/extract/extract_word_docids.rs +++ b/crates/milli/src/update/index_documents/extract/extract_word_docids.rs @@ -127,7 +127,8 @@ pub fn extract_word_docids( // merge all deletions let obkv = KvReaderDelAdd::from_slice(value); if let Some(value) = obkv.get(DelAdd::Deletion) { - let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid); + let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid) + || settings_diff.old.disabled_typos_terms.is_exact(w); buffer.clear(); let mut obkv = KvWriterDelAdd::new(&mut buffer); obkv.insert(DelAdd::Deletion, value)?; @@ -139,7 +140,8 @@ pub fn extract_word_docids( } // merge all additions if let Some(value) = obkv.get(DelAdd::Addition) { - let add_in_exact = settings_diff.new.exact_attributes.contains(&fid); + let add_in_exact = settings_diff.new.exact_attributes.contains(&fid) + || settings_diff.new.disabled_typos_terms.is_exact(w); buffer.clear(); let mut obkv = KvWriterDelAdd::new(&mut buffer); obkv.insert(DelAdd::Addition, value)?; diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index 87ea31942..6d575a98b 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -273,14 +273,11 @@ pub(crate) fn write_typed_chunk_into_index( unreachable!(); }; let clonable_word_docids = unsafe { as_cloneable_grenad(&word_docids_reader) }?; - let clonable_exact_word_docids = - unsafe { as_cloneable_grenad(&exact_word_docids_reader) }?; word_docids_builder.push(word_docids_reader.into_cursor()?); exact_word_docids_builder.push(exact_word_docids_reader.into_cursor()?); word_fid_docids_builder.push(word_fid_docids_reader.into_cursor()?); fst_merger_builder.push(clonable_word_docids.into_cursor()?); - fst_merger_builder.push(clonable_exact_word_docids.into_cursor()?); } let word_docids_merger = word_docids_builder.build(); diff --git a/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs b/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs index a085a89ae..046116939 100644 --- a/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs +++ b/crates/milli/src/update/new/extract/searchable/extract_word_docids.rs @@ -319,8 +319,11 @@ impl WordDocidsExtractors { let doc_alloc = &context.doc_alloc; let exact_attributes = index.exact_attributes(rtxn)?; - let is_exact_attribute = - |fname: &str| exact_attributes.iter().any(|attr| contained_in(fname, attr)); + let disabled_typos_terms = index.disabled_typos_terms(rtxn)?; + let is_exact = |fname: &str, word: &str| { + exact_attributes.iter().any(|attr| contained_in(fname, attr)) + || disabled_typos_terms.is_exact(word) + }; match document_change { DocumentChange::Deletion(inner) => { let mut token_fn = |fname: &str, fid, pos, word: &str| { @@ -328,7 +331,7 @@ impl WordDocidsExtractors { fid, pos, word, - is_exact_attribute(fname), + is_exact(fname, word), inner.docid(), doc_alloc, ) @@ -356,7 +359,7 @@ impl WordDocidsExtractors { fid, pos, word, - is_exact_attribute(fname), + is_exact(fname, word), inner.docid(), doc_alloc, ) @@ -372,7 +375,7 @@ impl WordDocidsExtractors { fid, pos, word, - is_exact_attribute(fname), + is_exact(fname, word), inner.docid(), doc_alloc, ) @@ -389,7 +392,7 @@ impl WordDocidsExtractors { fid, pos, word, - is_exact_attribute(fname), + is_exact(fname, word), inner.docid(), doc_alloc, ) diff --git a/crates/milli/src/update/new/indexer/mod.rs b/crates/milli/src/update/new/indexer/mod.rs index d2a88f4ff..2ea3c787e 100644 --- a/crates/milli/src/update/new/indexer/mod.rs +++ b/crates/milli/src/update/new/indexer/mod.rs @@ -9,6 +9,7 @@ pub use document_operation::{DocumentOperation, PayloadStats}; use hashbrown::HashMap; use heed::RwTxn; pub use partial_dump::PartialDump; +pub use post_processing::recompute_word_fst_from_word_docids_database; pub use update_by_function::UpdateByFunction; pub use write::ChannelCongestion; use write::{build_vectors, update_index, write_to_db}; diff --git a/crates/milli/src/update/new/indexer/post_processing.rs b/crates/milli/src/update/new/indexer/post_processing.rs index aace70cff..b5c89d0d9 100644 --- a/crates/milli/src/update/new/indexer/post_processing.rs +++ b/crates/milli/src/update/new/indexer/post_processing.rs @@ -131,6 +131,20 @@ fn compute_word_fst( } } +pub fn recompute_word_fst_from_word_docids_database(index: &Index, wtxn: &mut RwTxn) -> Result<()> { + let fst = fst::Set::default().map_data(std::borrow::Cow::Owned)?; + let mut word_fst_builder = WordFstBuilder::new(&fst)?; + let words = index.word_docids.iter(wtxn)?.remap_data_type::(); + for res in words { + let (word, _) = res?; + word_fst_builder.register_word(DelAdd::Addition, word.as_ref())?; + } + let (word_fst_mmap, _) = word_fst_builder.build(index, wtxn)?; + index.main.remap_types::().put(wtxn, WORDS_FST_KEY, &word_fst_mmap)?; + + Ok(()) +} + #[tracing::instrument(level = "trace", skip_all, target = "indexing::facet_search")] fn compute_facet_search_database( index: &Index, diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 317be1968..51d9aed27 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -17,6 +17,7 @@ use super::IndexerConfig; use crate::attribute_patterns::PatternMatch; use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::criterion::Criterion; +use crate::disabled_typos_terms::DisabledTyposTerms; use crate::error::UserError; use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder}; use crate::filterable_attributes_rules::match_faceted_field; @@ -169,6 +170,7 @@ pub struct Settings<'a, 't, 'i> { synonyms: Setting>>, primary_key: Setting, authorize_typos: Setting, + disable_on_numbers: Setting, min_word_len_two_typos: Setting, min_word_len_one_typo: Setting, exact_words: Setting>, @@ -207,6 +209,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { synonyms: Setting::NotSet, primary_key: Setting::NotSet, authorize_typos: Setting::NotSet, + disable_on_numbers: Setting::NotSet, exact_words: Setting::NotSet, min_word_len_two_typos: Setting::NotSet, min_word_len_one_typo: Setting::NotSet, @@ -354,6 +357,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { self.min_word_len_one_typo = Setting::Reset; } + pub fn set_disable_on_numbers(&mut self, disable_on_numbers: bool) { + self.disable_on_numbers = Setting::Set(disable_on_numbers); + } + + pub fn reset_disable_on_numbers(&mut self) { + self.disable_on_numbers = Setting::Reset; + } + pub fn set_exact_words(&mut self, words: BTreeSet) { self.exact_words = Setting::Set(words); } @@ -866,6 +877,24 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { Ok(()) } + fn update_disabled_typos_terms(&mut self) -> Result<()> { + let mut disabled_typos_terms = self.index.disabled_typos_terms(self.wtxn)?; + match self.disable_on_numbers { + Setting::Set(disable_on_numbers) => { + disabled_typos_terms.disable_on_numbers = disable_on_numbers; + } + Setting::Reset => { + self.index.delete_disabled_typos_terms(self.wtxn)?; + disabled_typos_terms.disable_on_numbers = + DisabledTyposTerms::default().disable_on_numbers; + } + Setting::NotSet => (), + } + + self.index.put_disabled_typos_terms(self.wtxn, &disabled_typos_terms)?; + Ok(()) + } + fn update_exact_words(&mut self) -> Result<()> { match self.exact_words { Setting::Set(ref mut words) => { @@ -1246,6 +1275,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { self.update_prefix_search()?; self.update_facet_search()?; self.update_localized_attributes_rules()?; + self.update_disabled_typos_terms()?; let embedding_config_updates = self.update_embedding_configs()?; @@ -1327,6 +1357,7 @@ impl InnerIndexSettingsDiff { || old_settings.prefix_search != new_settings.prefix_search || old_settings.localized_attributes_rules != new_settings.localized_attributes_rules + || old_settings.disabled_typos_terms != new_settings.disabled_typos_terms }; let cache_exact_attributes = old_settings.exact_attributes != new_settings.exact_attributes; @@ -1526,6 +1557,7 @@ pub(crate) struct InnerIndexSettings { pub user_defined_searchable_attributes: Option>, pub sortable_fields: HashSet, pub exact_attributes: HashSet, + pub disabled_typos_terms: DisabledTyposTerms, pub proximity_precision: ProximityPrecision, pub embedding_configs: EmbeddingConfigs, pub geo_fields_ids: Option<(FieldId, FieldId)>, @@ -1574,7 +1606,7 @@ impl InnerIndexSettings { .map(|fields| fields.into_iter().map(|f| f.to_string()).collect()); let builder = MetadataBuilder::from_index(index, rtxn)?; let fields_ids_map = FieldIdMapWithMetadata::new(fields_ids_map, builder); - + let disabled_typos_terms = index.disabled_typos_terms(rtxn)?; Ok(Self { stop_words, allowed_separators, @@ -1592,6 +1624,7 @@ impl InnerIndexSettings { geo_fields_ids, prefix_search, facet_search, + disabled_typos_terms, }) } diff --git a/crates/milli/src/update/test_settings.rs b/crates/milli/src/update/test_settings.rs index 00be0476a..2b9ee3a5e 100644 --- a/crates/milli/src/update/test_settings.rs +++ b/crates/milli/src/update/test_settings.rs @@ -896,6 +896,7 @@ fn test_correct_settings_init() { localized_attributes_rules, prefix_search, facet_search, + disable_on_numbers, } = settings; assert!(matches!(searchable_fields, Setting::NotSet)); assert!(matches!(displayed_fields, Setting::NotSet)); @@ -923,6 +924,7 @@ fn test_correct_settings_init() { assert!(matches!(localized_attributes_rules, Setting::NotSet)); assert!(matches!(prefix_search, Setting::NotSet)); assert!(matches!(facet_search, Setting::NotSet)); + assert!(matches!(disable_on_numbers, Setting::NotSet)); }) .unwrap(); } diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 7c8dcf64a..9f64ca0e3 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -1,12 +1,14 @@ mod v1_12; mod v1_13; mod v1_14; - +mod v1_15; use heed::RwTxn; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; +use v1_15::Latest_V1_14_To_Latest_V1_15; +use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; use crate::{Index, InternalError, Result}; @@ -23,12 +25,16 @@ trait UpgradeIndex { } /// Return true if the cached stats of the index must be regenerated -pub fn upgrade( +pub fn upgrade( wtxn: &mut RwTxn, index: &Index, db_version: (u32, u32, u32), + must_stop_processing: MSP, progress: Progress, -) -> Result { +) -> Result +where + MSP: Fn() -> bool + Sync, +{ let from = index.get_version(wtxn)?.unwrap_or(db_version); let upgrade_functions: &[&dyn UpgradeIndex] = &[ &V1_12_To_V1_12_3 {}, @@ -36,6 +42,10 @@ pub fn upgrade( &V1_13_0_To_V1_13_1 {}, &V1_13_1_To_Latest_V1_13 {}, &Latest_V1_13_To_Latest_V1_14 {}, + &Latest_V1_14_To_Latest_V1_15 {}, + // This is the last upgrade function, it will be called when the index is up to date. + // any other upgrade function should be added before this one. + &ToCurrentNoOp {}, ]; let start = match from { @@ -43,8 +53,9 @@ pub fn upgrade( (1, 12, 3..) => 1, (1, 13, 0) => 2, (1, 13, _) => 4, + (1, 14, _) => 5, // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. - (1, 14, _) => 4, + (1, 15, _) => 6, (major, minor, patch) => { return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) } @@ -56,6 +67,9 @@ pub fn upgrade( let mut current_version = from; let mut regenerate_stats = false; for (i, upgrade) in upgrade_path.iter().enumerate() { + if (must_stop_processing)() { + return Err(crate::Error::InternalError(InternalError::AbortedIndexation)); + } let target = upgrade.target_version(); progress.update_progress(VariableNameStep::::new( format!( @@ -77,3 +91,22 @@ pub fn upgrade( Ok(regenerate_stats) } + +#[allow(non_camel_case_types)] +struct ToCurrentNoOp {} + +impl UpgradeIndex for ToCurrentNoOp { + fn upgrade( + &self, + _wtxn: &mut RwTxn, + _index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) + } +} diff --git a/crates/milli/src/update/upgrade/v1_13.rs b/crates/milli/src/update/upgrade/v1_13.rs index 8e5e052bd..7f6608970 100644 --- a/crates/milli/src/update/upgrade/v1_13.rs +++ b/crates/milli/src/update/upgrade/v1_13.rs @@ -1,7 +1,6 @@ use heed::RwTxn; use super::UpgradeIndex; -use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::database_stats::DatabaseStats; use crate::progress::Progress; use crate::{make_enum_progress, Index, Result}; @@ -51,10 +50,6 @@ impl UpgradeIndex for V1_13_1_To_Latest_V1_13 { } fn target_version(&self) -> (u32, u32, u32) { - ( - VERSION_MAJOR.parse().unwrap(), - VERSION_MINOR.parse().unwrap(), - VERSION_PATCH.parse().unwrap(), - ) + (1, 13, 3) } } diff --git a/crates/milli/src/update/upgrade/v1_15.rs b/crates/milli/src/update/upgrade/v1_15.rs new file mode 100644 index 000000000..2c3cff355 --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_15.rs @@ -0,0 +1,35 @@ +use heed::RwTxn; + +use super::UpgradeIndex; +use crate::progress::Progress; +use crate::update::new::indexer::recompute_word_fst_from_word_docids_database; +use crate::{make_enum_progress, Index, Result}; + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_14_To_Latest_V1_15(); + +impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 { + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + _original: (u32, u32, u32), + progress: Progress, + ) -> Result { + // Recompute the word FST from the word docids database. + make_enum_progress! { + enum TypoTolerance { + RecomputeWordFst, + } + }; + + progress.update_progress(TypoTolerance::RecomputeWordFst); + recompute_word_fst_from_word_docids_database(index, wtxn)?; + + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 15, 0) + } +}