snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-26 13:36:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into all-cpus-in-import-dump

Browse Source
This commit is contained in:
nnethercott
2025-05-12 21:48:12 +02:00
parent 53f32a7dd7 d9a527854a
commit 75a7e40a27
74 changed files with 2402 additions and 1726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
.github/workflows/publish-docker-images.yml vendored
View File

@@ -104,3 +104,20 @@ jobs:
repository: meilisearch/meilisearch-cloud repository: meilisearch/meilisearch-cloud
event-type: cloud-docker-build event-type: cloud-docker-build
client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }' client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }'
# Send notification to Swarmia to notify of a deployment: https://app.swarmia.com
- name: Send deployment to Swarmia
if: github.event_name == 'push' && success()
run: |
JSON_STRING=$( jq --null-input --compact-output \
--arg version "${{ github.ref_name }}" \
--arg appName "meilisearch" \
--arg environment "production" \
--arg commitSha "${{ github.sha }}" \
--arg repositoryFullName "${{ github.repository }}" \
'{"version": $version, "appName": $appName, "environment": $environment, "commitSha": $commitSha, "repositoryFullName": $repositoryFullName}' )
curl -H "Authorization: ${{ secrets.SWARMIA_DEPLOYMENTS_AUTHORIZATION }}" \
-H "Content-Type: application/json" \
-d "$JSON_STRING" \
https://hook.swarmia.com/deployments

34
Cargo.lock generated
View File

@@ -486,7 +486,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2
[[package]] [[package]]
name = "benchmarks" name = "benchmarks"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bumpalo", "bumpalo",
@@ -677,7 +677,7 @@ dependencies = [
[[package]] [[package]]
name = "build-info" name = "build-info"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"time", "time",
@@ -1652,7 +1652,7 @@ dependencies = [
[[package]] [[package]]
name = "dump" name = "dump"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@@ -1854,7 +1854,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]] [[package]]
name = "file-store" name = "file-store"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"tempfile", "tempfile",
"thiserror 2.0.9", "thiserror 2.0.9",
@@ -1876,7 +1876,7 @@ dependencies = [
[[package]] [[package]]
name = "filter-parser" name = "filter-parser"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"insta", "insta",
"nom", "nom",
@@ -1896,7 +1896,7 @@ dependencies = [
[[package]] [[package]]
name = "flatten-serde-json" name = "flatten-serde-json"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"criterion", "criterion",
"serde_json", "serde_json",
@@ -2035,7 +2035,7 @@ dependencies = [
[[package]] [[package]]
name = "fuzzers" name = "fuzzers"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"arbitrary", "arbitrary",
"bumpalo", "bumpalo",
@@ -2738,7 +2738,7 @@ checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
[[package]] [[package]]
name = "index-scheduler" name = "index-scheduler"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"big_s", "big_s",
@@ -2947,7 +2947,7 @@ dependencies = [
[[package]] [[package]]
name = "json-depth-checker" name = "json-depth-checker"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"criterion", "criterion",
"serde_json", "serde_json",
@@ -3586,7 +3586,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]] [[package]]
name = "meili-snap" name = "meili-snap"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"insta", "insta",
"md5", "md5",
@@ -3595,7 +3595,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch" name = "meilisearch"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-http", "actix-http",
@@ -3687,7 +3687,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-auth" name = "meilisearch-auth"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"enum-iterator", "enum-iterator",
@@ -3706,7 +3706,7 @@ dependencies = [
[[package]] [[package]]
name = "meilisearch-types" name = "meilisearch-types"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"actix-web", "actix-web",
"anyhow", "anyhow",
@@ -3740,7 +3740,7 @@ dependencies = [
[[package]] [[package]]
name = "meilitool" name = "meilitool"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
@@ -3774,7 +3774,7 @@ dependencies = [
[[package]] [[package]]
name = "milli" name = "milli"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
"arroy", "arroy",
@@ -4287,7 +4287,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]] [[package]]
name = "permissive-json-pointer" name = "permissive-json-pointer"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"big_s", "big_s",
"serde_json", "serde_json",
@@ -6898,7 +6898,7 @@ dependencies = [
[[package]] [[package]]
name = "xtask" name = "xtask"
version = "1.14.0" version = "1.15.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"build-info", "build-info",

2
Cargo.toml
View File

@@ -22,7 +22,7 @@ members = [
] ]
[workspace.package] [workspace.package]
version = "1.14.0" version = "1.15.0"
authors = [ authors = [
"Quentin de Quelen <quentin@dequelen.me>", "Quentin de Quelen <quentin@dequelen.me>",
"Clément Renault <clement@meilisearch.com>", "Clément Renault <clement@meilisearch.com>",

4
README.md
View File

@@ -41,7 +41,7 @@
- [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos). - [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
- [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination. - [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
- [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs. - [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application. - [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos). See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos).
@@ -99,7 +99,7 @@ If you want to know more about the kind of data we collect and what we use it fo
## 📫 Get in touch! ## 📫 Get in touch!
Meilisearch is a search engine created by [Meili]([https://www.welcometothejungle.com/en/companies/meilisearch](https://www.meilisearch.com/careers)), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact) Meilisearch is a search engine created by [Meili](https://www.meilisearch.com/careers), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months. 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.

1
crates/dump/src/reader/compat/v5_to_v6.rs
View File

@@ -373,6 +373,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
}, },
disable_on_words: typo.disable_on_words.into(), disable_on_words: typo.disable_on_words.into(),
disable_on_attributes: typo.disable_on_attributes.into(), disable_on_attributes: typo.disable_on_attributes.into(),
disable_on_numbers: v6::Setting::NotSet,
}), }),
v5::Setting::Reset => v6::Setting::Reset, v5::Setting::Reset => v6::Setting::Reset,
v5::Setting::NotSet => v6::Setting::NotSet, v5::Setting::NotSet => v6::Setting::NotSet,

23
crates/index-scheduler/src/error.rs
View File

@@ -2,6 +2,7 @@ use std::fmt::Display;
use meilisearch_types::batches::BatchId; use meilisearch_types::batches::BatchId;
use meilisearch_types::error::{Code, ErrorCode}; use meilisearch_types::error::{Code, ErrorCode};
use meilisearch_types::milli::index::RollbackOutcome;
use meilisearch_types::tasks::{Kind, Status}; use meilisearch_types::tasks::{Kind, Status};
use meilisearch_types::{heed, milli}; use meilisearch_types::{heed, milli};
use thiserror::Error; use thiserror::Error;
@@ -150,8 +151,24 @@ pub enum Error {
CorruptedTaskQueue, CorruptedTaskQueue,
#[error(transparent)] #[error(transparent)]
DatabaseUpgrade(Box<Self>), DatabaseUpgrade(Box<Self>),
#[error("Failed to rollback for index `{index}`: {rollback_outcome} ")]
RollbackFailed { index: String, rollback_outcome: RollbackOutcome },
#[error(transparent)] #[error(transparent)]
UnrecoverableError(Box<Self>), UnrecoverableError(Box<Self>),
#[error("The index scheduler is in version v{}.{}.{}, but Meilisearch is in version v{}.{}.{}.\n - hint: start the correct version of Meilisearch, or consider updating your database. See also <https://www.meilisearch.com/docs/learn/update_and_migration/updating>",
index_scheduler_version.0, index_scheduler_version.1, index_scheduler_version.2,
package_version.0, package_version.1, package_version.2)]
IndexSchedulerVersionMismatch {
index_scheduler_version: (u32, u32, u32),
package_version: (u32, u32, u32),
},
#[error("Index `{index}` is in version v{}.{}.{}, but Meilisearch is in version v{}.{}.{}.\n - note: this is an internal error, please consider filing a bug report: <https://github.com/meilisearch/meilisearch/issues/new?template=bug_report.md>",
index_version.0, index_version.1, index_version.2, package_version.0, package_version.1, package_version.2)]
IndexVersionMismatch {
index: String,
index_version: (u32, u32, u32),
package_version: (u32, u32, u32),
},
#[error(transparent)] #[error(transparent)]
HeedTransaction(heed::Error), HeedTransaction(heed::Error),
@@ -209,6 +226,9 @@ impl Error {
| Error::CorruptedTaskQueue | Error::CorruptedTaskQueue
| Error::DatabaseUpgrade(_) | Error::DatabaseUpgrade(_)
| Error::UnrecoverableError(_) | Error::UnrecoverableError(_)
| Error::IndexSchedulerVersionMismatch { .. }
| Error::IndexVersionMismatch { .. }
| Error::RollbackFailed { .. }
| Error::HeedTransaction(_) => false, | Error::HeedTransaction(_) => false,
#[cfg(test)] #[cfg(test)]
Error::PlannedFailure => false, Error::PlannedFailure => false,
@@ -274,7 +294,10 @@ impl ErrorCode for Error {
Error::CorruptedTaskQueue => Code::Internal, Error::CorruptedTaskQueue => Code::Internal,
Error::CorruptedDump => Code::Internal, Error::CorruptedDump => Code::Internal,
Error::DatabaseUpgrade(_) => Code::Internal, Error::DatabaseUpgrade(_) => Code::Internal,
Error::RollbackFailed { .. } => Code::Internal,
Error::UnrecoverableError(_) => Code::Internal, Error::UnrecoverableError(_) => Code::Internal,
Error::IndexSchedulerVersionMismatch { .. } => Code::Internal,
Error::IndexVersionMismatch { .. } => Code::Internal,
Error::CreateBatch(_) => Code::Internal, Error::CreateBatch(_) => Code::Internal,
// This one should never be seen by the end user // This one should never be seen by the end user

46
crates/index-scheduler/src/index_mapper/mod.rs
View File

@@ -7,6 +7,7 @@ use meilisearch_types::heed::types::{SerdeJson, Str};
use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls}; use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls};
use meilisearch_types::milli; use meilisearch_types::milli;
use meilisearch_types::milli::database_stats::DatabaseStats; use meilisearch_types::milli::database_stats::DatabaseStats;
use meilisearch_types::milli::index::RollbackOutcome;
use meilisearch_types::milli::update::IndexerConfig; use meilisearch_types::milli::update::IndexerConfig;
use meilisearch_types::milli::{FieldDistribution, Index}; use meilisearch_types::milli::{FieldDistribution, Index};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -431,6 +432,51 @@ impl IndexMapper {
Ok(index) Ok(index)
} }
pub fn rollback_index(
&self,
rtxn: &RoTxn,
name: &str,
to: (u32, u32, u32),
) -> Result<RollbackOutcome> {
// remove any currently updating index to make sure that we aren't keeping a reference to the index somewhere
drop(self.currently_updating_index.write().unwrap().take());
let uuid = self
.index_mapping
.get(rtxn, name)?
.ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
// take the lock to make sure noone is messing with the indexes while we rollback
// this will block any search or other operation, but we are rollbacking so this is probably acceptable.
let mut index_map = self.index_map.write().unwrap();
'close_index: loop {
match index_map.get(&uuid) {
Available(_) => {
index_map.close_for_resize(&uuid, self.enable_mdb_writemap, 0);
// index should now be `Closing`; try again
continue;
}
// index already closed
Missing => break 'close_index,
// closing requested by this thread or another one; wait for closing to complete, then exit
Closing(closing_index) => {
if closing_index.wait_timeout(Duration::from_secs(100)).is_none() {
// release the lock so it doesn't get poisoned
drop(index_map);
panic!("cannot close index")
}
break;
}
BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
};
}
let index_path = self.base_path.join(uuid.to_string());
Index::rollback(milli::heed::EnvOpenOptions::new().read_txn_without_tls(), index_path, to)
.map_err(|err| crate::Error::from_milli(err, Some(name.to_string())))
}
/// Attempts `f` for each index that exists in the index mapper. /// Attempts `f` for each index that exists in the index mapper.
/// ///
/// It is preferable to use this function rather than a loop that opens all indexes, as a way to avoid having all indexes opened, /// It is preferable to use this function rather than a loop that opens all indexes, as a way to avoid having all indexes opened,

7
crates/index-scheduler/src/insta_snapshot.rs
View File

@@ -41,11 +41,8 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
let mut snap = String::new(); let mut snap = String::new();
let indx_sched_version = version.get_version(&rtxn).unwrap(); let indx_sched_version = version.get_version(&rtxn).unwrap();
let latest_version = ( let latest_version =
versioning::VERSION_MAJOR.parse().unwrap(), (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH);
versioning::VERSION_MINOR.parse().unwrap(),
versioning::VERSION_PATCH.parse().unwrap(),
);
if indx_sched_version != Some(latest_version) { if indx_sched_version != Some(latest_version) {
snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n")); snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n"));
} }

4
crates/index-scheduler/src/lib.rs
View File

@@ -398,9 +398,9 @@ impl IndexScheduler {
Ok(Ok(TickOutcome::StopProcessingForever)) => break, Ok(Ok(TickOutcome::StopProcessingForever)) => break,
Ok(Err(e)) => { Ok(Err(e)) => {
tracing::error!("{e}"); tracing::error!("{e}");
// Wait one second when an irrecoverable error occurs. // Wait when an irrecoverable error occurs.
if !e.is_recoverable() { if !e.is_recoverable() {
std::thread::sleep(Duration::from_secs(1)); std::thread::sleep(Duration::from_secs(10));
} }
} }
Err(_panic) => { Err(_panic) => {

1
crates/index-scheduler/src/processing.rs
View File

@@ -74,6 +74,7 @@ make_enum_progress! {
make_enum_progress! { make_enum_progress! {
pub enum TaskCancelationProgress { pub enum TaskCancelationProgress {
RetrievingTasks, RetrievingTasks,
CancelingUpgrade,
UpdatingTasks, UpdatingTasks,
} }
} }

44
crates/index-scheduler/src/scheduler/create_batch.rs
View File

@@ -423,7 +423,8 @@ impl IndexScheduler {
} }
/// Create the next batch to be processed; /// Create the next batch to be processed;
/// 1. We get the *last* task to cancel. /// 0. We get the *last* task to cancel.
/// 1. We get the tasks to upgrade.
/// 2. We get the *next* task to delete. /// 2. We get the *next* task to delete.
/// 3. We get the *next* snapshot to process. /// 3. We get the *next* snapshot to process.
/// 4. We get the *next* dump to process. /// 4. We get the *next* dump to process.
@@ -443,7 +444,20 @@ impl IndexScheduler {
let count_total_enqueued = enqueued.len(); let count_total_enqueued = enqueued.len();
let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?; let failed = &self.queue.tasks.get_status(rtxn, Status::Failed)?;
// 0. The priority over everything is to upgrade the instance // 0. we get the last task to cancel.
let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
if let Some(task_id) = to_cancel.max() {
let mut task =
self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
current_batch.processing(Some(&mut task));
current_batch.reason(BatchStopReason::TaskCannotBeBatched {
kind: Kind::TaskCancelation,
id: task_id,
});
return Ok(Some((Batch::TaskCancelation { task }, current_batch)));
}
// 1. We upgrade the instance
// There shouldn't be multiple upgrade tasks but just in case we're going to batch all of them at the same time // There shouldn't be multiple upgrade tasks but just in case we're going to batch all of them at the same time
let upgrade = self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)? & (enqueued | failed); let upgrade = self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)? & (enqueued | failed);
if !upgrade.is_empty() { if !upgrade.is_empty() {
@@ -459,17 +473,21 @@ impl IndexScheduler {
return Ok(Some((Batch::UpgradeDatabase { tasks }, current_batch))); return Ok(Some((Batch::UpgradeDatabase { tasks }, current_batch)));
} }
// 1. we get the last task to cancel. // check the version of the scheduler here.
let to_cancel = self.queue.tasks.get_kind(rtxn, Kind::TaskCancelation)? & enqueued; // if the version is not the current, refuse to batch any additional task.
if let Some(task_id) = to_cancel.max() { let version = self.version.get_version(rtxn)?;
let mut task = let package_version = (
self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?; meilisearch_types::versioning::VERSION_MAJOR,
current_batch.processing(Some(&mut task)); meilisearch_types::versioning::VERSION_MINOR,
current_batch.reason(BatchStopReason::TaskCannotBeBatched { meilisearch_types::versioning::VERSION_PATCH,
kind: Kind::TaskCancelation, );
id: task_id, if version != Some(package_version) {
}); return Err(Error::UnrecoverableError(Box::new(
return Ok(Some((Batch::TaskCancelation { task }, current_batch))); Error::IndexSchedulerVersionMismatch {
index_scheduler_version: version.unwrap_or((1, 12, 0)),
package_version,
},
)));
} }
// 2. we get the next task to delete // 2. we get the next task to delete

86
crates/index-scheduler/src/scheduler/process_batch.rs
View File

@@ -6,7 +6,8 @@ use meilisearch_types::batches::{BatchEnqueuedAt, BatchId};
use meilisearch_types::heed::{RoTxn, RwTxn}; use meilisearch_types::heed::{RoTxn, RwTxn};
use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::progress::{Progress, VariableNameStep};
use meilisearch_types::milli::{self, ChannelCongestion}; use meilisearch_types::milli::{self, ChannelCongestion};
use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
use milli::update::Settings as MilliSettings; use milli::update::Settings as MilliSettings;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
@@ -144,11 +145,22 @@ impl IndexScheduler {
self.index_mapper.index(&rtxn, &index_uid)? self.index_mapper.index(&rtxn, &index_uid)?
}; };
let mut index_wtxn = index.write_txn()?;
let index_version = index.get_version(&index_wtxn)?.unwrap_or((1, 12, 0));
let package_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
if index_version != package_version {
return Err(Error::IndexVersionMismatch {
index: index_uid,
index_version,
package_version,
});
}
// the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick
self.index_mapper self.index_mapper
.set_currently_updating_index(Some((index_uid.clone(), index.clone()))); .set_currently_updating_index(Some((index_uid.clone(), index.clone())));
let mut index_wtxn = index.write_txn()?;
let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?; let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?;
let (tasks, congestion) = let (tasks, congestion) =
self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?; self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?;
@@ -353,9 +365,11 @@ impl IndexScheduler {
let KindWithContent::UpgradeDatabase { from } = tasks.last().unwrap().kind else { let KindWithContent::UpgradeDatabase { from } = tasks.last().unwrap().kind else {
unreachable!(); unreachable!();
}; };
let ret = catch_unwind(AssertUnwindSafe(|| self.process_upgrade(from, progress))); let ret = catch_unwind(AssertUnwindSafe(|| self.process_upgrade(from, progress)));
match ret { match ret {
Ok(Ok(())) => (), Ok(Ok(())) => (),
Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask),
Ok(Err(e)) => return Err(Error::DatabaseUpgrade(Box::new(e))), Ok(Err(e)) => return Err(Error::DatabaseUpgrade(Box::new(e))),
Err(e) => { Err(e) => {
let msg = match e.downcast_ref::<&'static str>() { let msg = match e.downcast_ref::<&'static str>() {
@@ -653,17 +667,79 @@ impl IndexScheduler {
progress: &Progress, progress: &Progress,
) -> Result<Vec<Task>> { ) -> Result<Vec<Task>> {
progress.update_progress(TaskCancelationProgress::RetrievingTasks); progress.update_progress(TaskCancelationProgress::RetrievingTasks);
let mut tasks_to_cancel = RoaringBitmap::new();
let enqueued_tasks = &self.queue.tasks.get_status(rtxn, Status::Enqueued)?;
// 0. Check if any upgrade task was matched.
// If so, we cancel all the failed or enqueued upgrade tasks.
let upgrade_tasks = &self.queue.tasks.get_kind(rtxn, Kind::UpgradeDatabase)?;
let is_canceling_upgrade = !matched_tasks.is_disjoint(upgrade_tasks);
if is_canceling_upgrade {
let failed_tasks = self.queue.tasks.get_status(rtxn, Status::Failed)?;
tasks_to_cancel |= upgrade_tasks & (enqueued_tasks | failed_tasks);
}
// 1. Remove from this list the tasks that we are not allowed to cancel // 1. Remove from this list the tasks that we are not allowed to cancel
// Notice that only the _enqueued_ ones are cancelable and we should // Notice that only the _enqueued_ ones are cancelable and we should
// have already aborted the indexation of the _processing_ ones // have already aborted the indexation of the _processing_ ones
let cancelable_tasks = self.queue.tasks.get_status(rtxn, Status::Enqueued)?; tasks_to_cancel |= enqueued_tasks & matched_tasks;
let tasks_to_cancel = cancelable_tasks & matched_tasks;
// 2. If we're canceling an upgrade, attempt the rollback
if let Some(latest_upgrade_task) = (&tasks_to_cancel & upgrade_tasks).max() {
progress.update_progress(TaskCancelationProgress::CancelingUpgrade);
let task = self.queue.tasks.get_task(rtxn, latest_upgrade_task)?.unwrap();
let Some(Details::UpgradeDatabase { from, to }) = task.details else {
unreachable!("wrong details for upgrade task {latest_upgrade_task}")
};
// check that we are rollbacking an upgrade to the current Meilisearch
let bin_major: u32 = meilisearch_types::versioning::VERSION_MAJOR;
let bin_minor: u32 = meilisearch_types::versioning::VERSION_MINOR;
let bin_patch: u32 = meilisearch_types::versioning::VERSION_PATCH;
if to == (bin_major, bin_minor, bin_patch) {
tracing::warn!(
"Rollbacking from v{}.{}.{} to v{}.{}.{}",
to.0,
to.1,
to.2,
from.0,
from.1,
from.2
);
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
self.process_rollback(from, progress)
})) {
Ok(Ok(())) => {}
Ok(Err(err)) => return Err(Error::DatabaseUpgrade(Box::new(err))),
Err(e) => {
let msg = match e.downcast_ref::<&'static str>() {
Some(s) => *s,
None => match e.downcast_ref::<String>() {
Some(s) => &s[..],
None => "Box<dyn Any>",
},
};
return Err(Error::DatabaseUpgrade(Box::new(Error::ProcessBatchPanicked(
msg.to_string(),
))));
}
}
} else {
tracing::debug!(
"Not rollbacking an upgrade targetting the earlier version v{}.{}.{}",
bin_major,
bin_minor,
bin_patch
)
}
}
// 3. We now have a list of tasks to cancel, cancel them
let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32); let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32);
progress.update_progress(progress_obj); progress.update_progress(progress_obj);
// 2. We now have a list of tasks to cancel, cancel them
let mut tasks = self.queue.tasks.get_existing_tasks( let mut tasks = self.queue.tasks.get_existing_tasks(
rtxn, rtxn,
tasks_to_cancel.iter().inspect(|_| { tasks_to_cancel.iter().inspect(|_| {

45
crates/index-scheduler/src/scheduler/process_upgrade/mod.rs
View File

@@ -12,10 +12,14 @@ impl IndexScheduler {
#[cfg(test)] #[cfg(test)]
self.maybe_fail(crate::test_utils::FailureLocation::ProcessUpgrade)?; self.maybe_fail(crate::test_utils::FailureLocation::ProcessUpgrade)?;
enum UpgradeIndex {}
let indexes = self.index_names()?; let indexes = self.index_names()?;
for (i, uid) in indexes.iter().enumerate() { for (i, uid) in indexes.iter().enumerate() {
let must_stop_processing = self.scheduler.must_stop_processing.clone();
if must_stop_processing.get() {
return Err(Error::AbortedTask);
}
progress.update_progress(VariableNameStep::<UpgradeIndex>::new( progress.update_progress(VariableNameStep::<UpgradeIndex>::new(
format!("Upgrading index `{uid}`"), format!("Upgrading index `{uid}`"),
i as u32, i as u32,
@@ -27,6 +31,7 @@ impl IndexScheduler {
&mut index_wtxn, &mut index_wtxn,
&index, &index,
db_version, db_version,
|| must_stop_processing.get(),
progress.clone(), progress.clone(),
) )
.map_err(|e| Error::from_milli(e, Some(uid.to_string())))?; .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
@@ -46,4 +51,42 @@ impl IndexScheduler {
Ok(()) Ok(())
} }
pub fn process_rollback(&self, db_version: (u32, u32, u32), progress: &Progress) -> Result<()> {
let mut wtxn = self.env.write_txn()?;
tracing::info!(?db_version, "roll back index scheduler version");
self.version.set_version(&mut wtxn, db_version)?;
let db_path = self.scheduler.version_file_path.parent().unwrap();
wtxn.commit()?;
let indexes = self.index_names()?;
tracing::info!("roll backing all indexes");
for (i, uid) in indexes.iter().enumerate() {
progress.update_progress(VariableNameStep::<UpgradeIndex>::new(
format!("Rollbacking index `{uid}`"),
i as u32,
indexes.len() as u32,
));
let index_schd_rtxn = self.env.read_txn()?;
let rollback_outcome =
self.index_mapper.rollback_index(&index_schd_rtxn, uid, db_version)?;
if !rollback_outcome.succeeded() {
return Err(crate::Error::RollbackFailed { index: uid.clone(), rollback_outcome });
}
}
tracing::info!(?db_path, ?db_version, "roll back version file");
meilisearch_types::versioning::create_version_file(
db_path,
db_version.0,
db_version.1,
db_version.2,
)?;
Ok(())
}
} }
enum UpgradeIndex {}

4
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
[timestamp] [4,] [timestamp] [4,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } 0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", }
1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "task with id 1 of type `indexCreation` cannot be batched", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "task with id 1 of type `indexCreation` cannot be batched", }
2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 2 of type `indexCreation` cannot be batched", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 2 of type `indexCreation` cannot be batched", }
3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 3 of type `indexCreation` cannot be batched", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "task with id 3 of type `indexCreation` cannot be batched", }

2
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
enqueued [0,] enqueued [0,]

2
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:

4
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Status: ### Status:
@@ -37,7 +37,7 @@ catto [1,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } 0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
---------------------------------------------------------------------- ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } 0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

4
crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap
View File

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
[] []
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Tasks: ### All Tasks:
0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 14, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
[timestamp] [0,] [timestamp] [0,]
---------------------------------------------------------------------- ----------------------------------------------------------------------
### All Batches: ### All Batches:
0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.14.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", } 0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task", }
---------------------------------------------------------------------- ----------------------------------------------------------------------
### Batch to tasks mapping: ### Batch to tasks mapping:
0 [0,] 0 [0,]

8
crates/index-scheduler/src/test_utils.rs
View File

@@ -114,12 +114,8 @@ impl IndexScheduler {
auto_upgrade: true, // Don't cost much and will ensure the happy path works auto_upgrade: true, // Don't cost much and will ensure the happy path works
embedding_cache_cap: 10, embedding_cache_cap: 10,
}; };
let version = configuration(&mut options).unwrap_or_else(|| { let version = configuration(&mut options).unwrap_or({
( (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH)
versioning::VERSION_MAJOR.parse().unwrap(),
versioning::VERSION_MINOR.parse().unwrap(),
versioning::VERSION_PATCH.parse().unwrap(),
)
}); });
std::fs::create_dir_all(&options.auth_path).unwrap(); std::fs::create_dir_all(&options.auth_path).unwrap();

13
crates/index-scheduler/src/upgrade/mod.rs
View File

@@ -28,12 +28,17 @@ pub fn upgrade_index_scheduler(
let current_minor = to.1; let current_minor = to.1;
let current_patch = to.2; let current_patch = to.2;
let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[&ToCurrentNoOp {}]; let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[
// This is the last upgrade function, it will be called when the index is up to date.
// any other upgrade function should be added before this one.
&ToCurrentNoOp {},
];
let start = match from { let start = match from {
(1, 12, _) => 0, (1, 12, _) => 0,
(1, 13, _) => 0, (1, 13, _) => 0,
(1, 14, _) => 0, (1, 14, _) => 0,
(1, 15, _) => 0,
(major, minor, patch) => { (major, minor, patch) => {
if major > current_major if major > current_major
|| (major == current_major && minor > current_minor) || (major == current_major && minor > current_minor)
@@ -104,10 +109,6 @@ impl UpgradeIndexScheduler for ToCurrentNoOp {
} }
fn target_version(&self) -> (u32, u32, u32) { fn target_version(&self) -> (u32, u32, u32) {
( (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
VERSION_MAJOR.parse().unwrap(),
VERSION_MINOR.parse().unwrap(),
VERSION_PATCH.parse().unwrap(),
)
} }
} }

12
crates/index-scheduler/src/versioning.rs
View File

@@ -39,9 +39,9 @@ impl Versioning {
} }
pub fn set_current_version(&self, wtxn: &mut RwTxn) -> Result<(), heed::Error> { pub fn set_current_version(&self, wtxn: &mut RwTxn) -> Result<(), heed::Error> {
let major = versioning::VERSION_MAJOR.parse().unwrap(); let major = versioning::VERSION_MAJOR;
let minor = versioning::VERSION_MINOR.parse().unwrap(); let minor = versioning::VERSION_MINOR;
let patch = versioning::VERSION_PATCH.parse().unwrap(); let patch = versioning::VERSION_PATCH;
self.set_version(wtxn, (major, minor, patch)) self.set_version(wtxn, (major, minor, patch))
} }
@@ -64,9 +64,9 @@ impl Versioning {
}; };
wtxn.commit()?; wtxn.commit()?;
let bin_major: u32 = versioning::VERSION_MAJOR.parse().unwrap(); let bin_major: u32 = versioning::VERSION_MAJOR;
let bin_minor: u32 = versioning::VERSION_MINOR.parse().unwrap(); let bin_minor: u32 = versioning::VERSION_MINOR;
let bin_patch: u32 = versioning::VERSION_PATCH.parse().unwrap(); let bin_patch: u32 = versioning::VERSION_PATCH;
let to = (bin_major, bin_minor, bin_patch); let to = (bin_major, bin_minor, bin_patch);
if from != to { if from != to {

4
crates/meilisearch-types/src/batch_view.rs
View File

@@ -22,7 +22,7 @@ pub struct BatchView {
#[serde(with = "time::serde::rfc3339::option", default)] #[serde(with = "time::serde::rfc3339::option", default)]
pub finished_at: Option<OffsetDateTime>, pub finished_at: Option<OffsetDateTime>,
#[serde(default = "meilisearch_types::batches::default_stop_reason")] #[serde(default = "meilisearch_types::batches::default_stop_reason")]
pub batcher_stopped_because: String, pub batch_creation_complete: String,
} }
impl BatchView { impl BatchView {
@@ -35,7 +35,7 @@ impl BatchView {
duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at), duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at),
started_at: batch.started_at, started_at: batch.started_at,
finished_at: batch.finished_at, finished_at: batch.finished_at,
batcher_stopped_because: batch.stop_reason.clone(), batch_creation_complete: batch.stop_reason.clone(),
} }
} }
} }

13
crates/meilisearch-types/src/settings.rs
View File

@@ -8,6 +8,7 @@ use std::str::FromStr;
use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef}; use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
use fst::IntoStreamer; use fst::IntoStreamer;
use milli::disabled_typos_terms::DisabledTyposTerms;
use milli::index::{IndexEmbeddingConfig, PrefixSearch}; use milli::index::{IndexEmbeddingConfig, PrefixSearch};
use milli::proximity::ProximityPrecision; use milli::proximity::ProximityPrecision;
use milli::update::Setting; use milli::update::Setting;
@@ -104,6 +105,10 @@ pub struct TypoSettings {
#[deserr(default)] #[deserr(default)]
#[schema(value_type = Option<BTreeSet<String>>, example = json!(["uuid", "url"]))] #[schema(value_type = Option<BTreeSet<String>>, example = json!(["uuid", "url"]))]
pub disable_on_attributes: Setting<BTreeSet<String>>, pub disable_on_attributes: Setting<BTreeSet<String>>,
#[serde(default, skip_serializing_if = "Setting::is_not_set")]
#[deserr(default)]
#[schema(value_type = Option<bool>, example = json!(true))]
pub disable_on_numbers: Setting<bool>,
} }
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
@@ -701,6 +706,12 @@ pub fn apply_settings_to_builder(
Setting::Reset => builder.reset_exact_attributes(), Setting::Reset => builder.reset_exact_attributes(),
Setting::NotSet => (), Setting::NotSet => (),
} }
match value.disable_on_numbers {
Setting::Set(val) => builder.set_disable_on_numbers(val),
Setting::Reset => builder.reset_disable_on_numbers(),
Setting::NotSet => (),
}
} }
Setting::Reset => { Setting::Reset => {
// all typo settings need to be reset here. // all typo settings need to be reset here.
@@ -826,12 +837,14 @@ pub fn settings(
}; };
let disabled_attributes = index.exact_attributes(rtxn)?.into_iter().map(String::from).collect(); let disabled_attributes = index.exact_attributes(rtxn)?.into_iter().map(String::from).collect();
let DisabledTyposTerms { disable_on_numbers } = index.disabled_typos_terms(rtxn)?;
let typo_tolerance = TypoSettings { let typo_tolerance = TypoSettings {
enabled: Setting::Set(index.authorize_typos(rtxn)?), enabled: Setting::Set(index.authorize_typos(rtxn)?),
min_word_size_for_typos: Setting::Set(min_typo_word_len), min_word_size_for_typos: Setting::Set(min_typo_word_len),
disable_on_words: Setting::Set(disabled_words), disable_on_words: Setting::Set(disabled_words),
disable_on_attributes: Setting::Set(disabled_attributes), disable_on_attributes: Setting::Set(disabled_attributes),
disable_on_numbers: Setting::Set(disable_on_numbers),
}; };
let faceting = FacetingSettings { let faceting = FacetingSettings {

18
crates/meilisearch-types/src/tasks.rs
View File

@@ -272,9 +272,9 @@ impl KindWithContent {
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
from: (from.0, from.1, from.2), from: (from.0, from.1, from.2),
to: ( to: (
versioning::VERSION_MAJOR.parse().unwrap(), versioning::VERSION_MAJOR,
versioning::VERSION_MINOR.parse().unwrap(), versioning::VERSION_MINOR,
versioning::VERSION_PATCH.parse().unwrap(), versioning::VERSION_PATCH,
), ),
}), }),
} }
@@ -338,9 +338,9 @@ impl KindWithContent {
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
from: *from, from: *from,
to: ( to: (
versioning::VERSION_MAJOR.parse().unwrap(), versioning::VERSION_MAJOR,
versioning::VERSION_MINOR.parse().unwrap(), versioning::VERSION_MINOR,
versioning::VERSION_PATCH.parse().unwrap(), versioning::VERSION_PATCH,
), ),
}), }),
} }
@@ -386,9 +386,9 @@ impl From<&KindWithContent> for Option<Details> {
KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase { KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
from: *from, from: *from,
to: ( to: (
versioning::VERSION_MAJOR.parse().unwrap(), versioning::VERSION_MAJOR,
versioning::VERSION_MINOR.parse().unwrap(), versioning::VERSION_MINOR,
versioning::VERSION_PATCH.parse().unwrap(), versioning::VERSION_PATCH,
), ),
}), }),
} }

10
crates/meilisearch-types/src/versioning.rs
View File

@@ -8,9 +8,7 @@ use tempfile::NamedTempFile;
/// The name of the file that contains the version of the database. /// The name of the file that contains the version of the database.
pub const VERSION_FILE_NAME: &str = "VERSION"; pub const VERSION_FILE_NAME: &str = "VERSION";
pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); pub use milli::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
/// Persists the version of the current Meilisearch binary to a VERSION file /// Persists the version of the current Meilisearch binary to a VERSION file
pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> { pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> {
@@ -19,9 +17,9 @@ pub fn create_current_version_file(db_path: &Path) -> anyhow::Result<()> {
pub fn create_version_file( pub fn create_version_file(
db_path: &Path, db_path: &Path,
major: &str, major: u32,
minor: &str, minor: u32,
patch: &str, patch: u32,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let version_path = db_path.join(VERSION_FILE_NAME); let version_path = db_path.join(VERSION_FILE_NAME);
// In order to persist the file later we must create it in the `data.ms` and not in `/tmp` // In order to persist the file later we must create it in the `data.ms` and not in `/tmp`

5
crates/meilisearch/src/lib.rs
View File

@@ -236,10 +236,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
auto_upgrade: opt.experimental_dumpless_upgrade, auto_upgrade: opt.experimental_dumpless_upgrade,
embedding_cache_cap: opt.experimental_embedding_cache_entries, embedding_cache_cap: opt.experimental_embedding_cache_entries,
}; };
let bin_major: u32 = VERSION_MAJOR.parse().unwrap(); let binary_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
let bin_minor: u32 = VERSION_MINOR.parse().unwrap();
let bin_patch: u32 = VERSION_PATCH.parse().unwrap();
let binary_version = (bin_major, bin_minor, bin_patch);
let empty_db = is_empty_db(&opt.db_path); let empty_db = is_empty_db(&opt.db_path);
let (index_scheduler, auth_controller) = if let Some(ref snapshot_path) = opt.import_snapshot { let (index_scheduler, auth_controller) = if let Some(ref snapshot_path) = opt.import_snapshot {

42
crates/meilisearch/tests/batches/mod.rs
View File

@@ -310,7 +310,7 @@ async fn test_summarized_document_addition_or_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
@@ -353,7 +353,7 @@ async fn test_summarized_document_addition_or_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
} }
@@ -398,7 +398,7 @@ async fn test_summarized_delete_documents_by_batch() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
@@ -440,7 +440,7 @@ async fn test_summarized_delete_documents_by_batch() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
} }
@@ -488,7 +488,7 @@ async fn test_summarized_delete_documents_by_filter() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
@@ -532,7 +532,7 @@ async fn test_summarized_delete_documents_by_filter() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
@@ -576,7 +576,7 @@ async fn test_summarized_delete_documents_by_filter() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
} }
@@ -622,7 +622,7 @@ async fn test_summarized_delete_document_by_id() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
@@ -664,7 +664,7 @@ async fn test_summarized_delete_document_by_id() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
} }
@@ -731,7 +731,7 @@ async fn test_summarized_settings_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
} }
"###); "###);
} }
@@ -773,7 +773,7 @@ async fn test_summarized_index_creation() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 0 of type `indexCreation` cannot be batched" "batchCreationComplete": "task with id 0 of type `indexCreation` cannot be batched"
} }
"###); "###);
@@ -812,7 +812,7 @@ async fn test_summarized_index_creation() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 1 of type `indexCreation` cannot be batched" "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched"
} }
"###); "###);
} }
@@ -964,7 +964,7 @@ async fn test_summarized_index_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 0 of type `indexUpdate` cannot be batched" "batchCreationComplete": "task with id 0 of type `indexUpdate` cannot be batched"
} }
"###); "###);
@@ -1003,7 +1003,7 @@ async fn test_summarized_index_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 1 of type `indexUpdate` cannot be batched" "batchCreationComplete": "task with id 1 of type `indexUpdate` cannot be batched"
} }
"###); "###);
@@ -1043,7 +1043,7 @@ async fn test_summarized_index_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 3 of type `indexUpdate` cannot be batched" "batchCreationComplete": "task with id 3 of type `indexUpdate` cannot be batched"
} }
"###); "###);
@@ -1082,7 +1082,7 @@ async fn test_summarized_index_update() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 4 of type `indexUpdate` cannot be batched" "batchCreationComplete": "task with id 4 of type `indexUpdate` cannot be batched"
} }
"###); "###);
} }
@@ -1134,7 +1134,7 @@ async fn test_summarized_index_swap() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 0 of type `indexSwap` cannot be batched" "batchCreationComplete": "task with id 0 of type `indexSwap` cannot be batched"
} }
"###); "###);
@@ -1177,7 +1177,7 @@ async fn test_summarized_index_swap() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 1 of type `indexCreation` cannot be batched" "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched"
} }
"###); "###);
} }
@@ -1224,7 +1224,7 @@ async fn test_summarized_batch_cancelation() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 1 of type `taskCancelation` cannot be batched" "batchCreationComplete": "task with id 1 of type `taskCancelation` cannot be batched"
} }
"###); "###);
} }
@@ -1271,7 +1271,7 @@ async fn test_summarized_batch_deletion() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "a batch of tasks of type `taskDeletion` cannot be batched with any other type of task" "batchCreationComplete": "a batch of tasks of type `taskDeletion` cannot be batched with any other type of task"
} }
"###); "###);
} }
@@ -1313,7 +1313,7 @@ async fn test_summarized_dump_creation() {
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "task with id 0 of type `dumpCreation` cannot be batched" "batchCreationComplete": "task with id 0 of type `dumpCreation` cannot be batched"
} }
"###); "###);
} }

42
crates/meilisearch/tests/dumps/mod.rs
View File

@@ -87,7 +87,8 @@ async fn import_dump_v1_movie_raw() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -260,7 +261,8 @@ async fn import_dump_v1_movie_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -432,7 +434,8 @@ async fn import_dump_v1_rubygems_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -590,7 +593,8 @@ async fn import_dump_v2_movie_raw() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -760,7 +764,8 @@ async fn import_dump_v2_movie_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -929,7 +934,8 @@ async fn import_dump_v2_rubygems_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1087,7 +1093,8 @@ async fn import_dump_v3_movie_raw() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1257,7 +1264,8 @@ async fn import_dump_v3_movie_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1426,7 +1434,8 @@ async fn import_dump_v3_rubygems_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1584,7 +1593,8 @@ async fn import_dump_v4_movie_raw() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1754,7 +1764,8 @@ async fn import_dump_v4_movie_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -1923,7 +1934,8 @@ async fn import_dump_v4_rubygems_with_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -2212,7 +2224,8 @@ async fn import_dump_v6_containing_experimental_features() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -2444,7 +2457,8 @@ async fn generate_and_import_dump_containing_vectors() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,

6
crates/meilisearch/tests/dumps/snapshots/mod.rs/import_dump_v6_containing_batches_and_enqueued_tasks/batches.snap
View File

@@ -27,7 +27,7 @@ source: crates/meilisearch/tests/dumps/mod.rs
"duration": "[date]", "duration": "[date]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "batched all enqueued tasks" "batchCreationComplete": "batched all enqueued tasks"
}, },
{ {
"uid": 1, "uid": 1,
@@ -51,7 +51,7 @@ source: crates/meilisearch/tests/dumps/mod.rs
"duration": "PT0.144827890S", "duration": "PT0.144827890S",
"startedAt": "2025-02-04T10:15:21.275640274Z", "startedAt": "2025-02-04T10:15:21.275640274Z",
"finishedAt": "2025-02-04T10:15:21.420468164Z", "finishedAt": "2025-02-04T10:15:21.420468164Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 0, "uid": 0,
@@ -72,7 +72,7 @@ source: crates/meilisearch/tests/dumps/mod.rs
"duration": "PT0.032902186S", "duration": "PT0.032902186S",
"startedAt": "2025-02-04T10:14:43.559526162Z", "startedAt": "2025-02-04T10:14:43.559526162Z",
"finishedAt": "2025-02-04T10:14:43.592428348Z", "finishedAt": "2025-02-04T10:14:43.592428348Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 3, "total": 3,

90
crates/meilisearch/tests/search/mod.rs
View File

@@ -1976,3 +1976,93 @@ async fn change_facet_casing() {
}) })
.await; .await;
} }
#[actix_rt::test]
async fn test_exact_typos_terms() {
let documents = json!([
{
"id": 0,
"title": "The zeroth document 1298484",
},
{
"id": 1,
"title": "The first document 234342",
"nested": {
"object": "field 22231",
"machin": "bidule 23443.32111",
},
},
{
"id": 2,
"title": "The second document 3398499",
"nested": [
"array",
{
"object": "field 23245121,23223",
},
{
"prout": "truc 123980612321",
"machin": "lol 12345645333447879",
},
],
},
{
"id": 3,
"title": "The third document 12333",
"nested": "I lied 98878",
},
]);
// Test prefix search
test_settings_documents_indexing_swapping_and_search(
&documents,
&json!({
"searchableAttributes": ["title", "nested.object", "nested.machin"],
"typoTolerance": {
"enabled": true,
"disableOnNumbers": true
}
}),
&json!({"q": "12345"}),
|response, code| {
assert_eq!(code, 200, "{}", response);
snapshot!(json_string!(response["hits"]), @r###"
[
{
"id": 2,
"title": "The second document 3398499",
"nested": [
"array",
{
"object": "field 23245121,23223"
},
{
"prout": "truc 123980612321",
"machin": "lol 12345645333447879"
}
]
}
]
"###);
},
)
.await;
// Test typo search
test_settings_documents_indexing_swapping_and_search(
&documents,
&json!({
"searchableAttributes": ["title", "nested.object", "nested.machin"],
"typoTolerance": {
"enabled": true,
"disableOnNumbers": true
}
}),
&json!({"q": "123457"}),
|response, code| {
assert_eq!(code, 200, "{}", response);
snapshot!(json_string!(response["hits"]), @r###"[]"###);
},
)
.await;
}

2
crates/meilisearch/tests/settings/errors.rs
View File

@@ -274,7 +274,7 @@ async fn settings_bad_typo_tolerance() {
snapshot!(code, @"400 Bad Request"); snapshot!(code, @"400 Bad Request");
snapshot!(json_string!(response), @r###" snapshot!(json_string!(response), @r###"
{ {
"message": "Unknown field `typoTolerance`: expected one of `enabled`, `minWordSizeForTypos`, `disableOnWords`, `disableOnAttributes`", "message": "Unknown field `typoTolerance`: expected one of `enabled`, `minWordSizeForTypos`, `disableOnWords`, `disableOnAttributes`, `disableOnNumbers`",
"code": "invalid_settings_typo_tolerance", "code": "invalid_settings_typo_tolerance",
"type": "invalid_request", "type": "invalid_request",
"link": "https://docs.meilisearch.com/errors#invalid_settings_typo_tolerance" "link": "https://docs.meilisearch.com/errors#invalid_settings_typo_tolerance"

9
crates/meilisearch/tests/settings/get_settings.rs
View File

@@ -179,7 +179,7 @@ test_setting_routes!(
{ {
setting: typo_tolerance, setting: typo_tolerance,
update_verb: patch, update_verb: patch,
default_value: {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": []} default_value: {"enabled": true, "minWordSizeForTypos": {"oneTypo": 5, "twoTypos": 9}, "disableOnWords": [], "disableOnAttributes": [], "disableOnNumbers": false}
}, },
); );
@@ -276,7 +276,7 @@ async fn secrets_are_hidden_in_settings() {
let (response, code) = index.settings().await; let (response, code) = index.settings().await;
meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(code, @"200 OK");
meili_snap::snapshot!(meili_snap::json_string!(response), @r#" meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
{ {
"displayedAttributes": [ "displayedAttributes": [
"*" "*"
@@ -308,7 +308,8 @@ async fn secrets_are_hidden_in_settings() {
"twoTypos": 9 "twoTypos": 9
}, },
"disableOnWords": [], "disableOnWords": [],
"disableOnAttributes": [] "disableOnAttributes": [],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 100, "maxValuesPerFacet": 100,
@@ -337,7 +338,7 @@ async fn secrets_are_hidden_in_settings() {
"facetSearch": true, "facetSearch": true,
"prefixSearch": "indexingTime" "prefixSearch": "indexingTime"
} }
"#); "###);
let (response, code) = server.get_task(settings_update_uid).await; let (response, code) = server.get_task(settings_update_uid).await;
meili_snap::snapshot!(code, @"200 OK"); meili_snap::snapshot!(code, @"200 OK");

6
crates/meilisearch/tests/upgrade/mod.rs
View File

@@ -43,7 +43,7 @@ async fn version_too_old() {
std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.14.0"); snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.15.0");
} }
#[actix_rt::test] #[actix_rt::test]
@@ -54,11 +54,11 @@ async fn version_requires_downgrade() {
std::fs::create_dir_all(&db_path).unwrap(); std::fs::create_dir_all(&db_path).unwrap();
let major = meilisearch_types::versioning::VERSION_MAJOR; let major = meilisearch_types::versioning::VERSION_MAJOR;
let minor = meilisearch_types::versioning::VERSION_MINOR; let minor = meilisearch_types::versioning::VERSION_MINOR;
let patch = meilisearch_types::versioning::VERSION_PATCH.parse::<u32>().unwrap() + 1; let patch = meilisearch_types::versioning::VERSION_PATCH + 1;
std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
snapshot!(err, @"Database version 1.14.1 is higher than the Meilisearch version 1.14.0. Downgrade is not supported"); snapshot!(err, @"Database version 1.15.1 is higher than the Meilisearch version 1.15.0. Downgrade is not supported");
} }
#[actix_rt::test] #[actix_rt::test]

4
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap
View File

@@ -1,6 +1,5 @@
--- ---
source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
snapshot_kind: text
--- ---
{ {
"displayedAttributes": [ "displayedAttributes": [
@@ -49,7 +48,8 @@ snapshot_kind: text
], ],
"disableOnAttributes": [ "disableOnAttributes": [
"surname" "surname"
] ],
"disableOnNumbers": false
}, },
"faceting": { "faceting": {
"maxValuesPerFacet": 99, "maxValuesPerFacet": 99,

42
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null, "progress": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
@@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task"
}, },
{ {
"uid": 23, "uid": 23,
@@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.004146631S", "duration": "PT0.004146631S",
"startedAt": "2025-01-23T11:38:57.012591321Z", "startedAt": "2025-01-23T11:38:57.012591321Z",
"finishedAt": "2025-01-23T11:38:57.016737952Z", "finishedAt": "2025-01-23T11:38:57.016737952Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 22, "uid": 22,
@@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.102738497S", "duration": "PT0.102738497S",
"startedAt": "2025-01-23T11:36:22.551906856Z", "startedAt": "2025-01-23T11:36:22.551906856Z",
"finishedAt": "2025-01-23T11:36:22.654645353Z", "finishedAt": "2025-01-23T11:36:22.654645353Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 21, "uid": 21,
@@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.005108474S", "duration": "PT0.005108474S",
"startedAt": "2025-01-23T11:36:04.132670526Z", "startedAt": "2025-01-23T11:36:04.132670526Z",
"finishedAt": "2025-01-23T11:36:04.137779Z", "finishedAt": "2025-01-23T11:36:04.137779Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 20, "uid": 20,
@@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.027954894S", "duration": "PT0.027954894S",
"startedAt": "2025-01-23T11:35:53.631082795Z", "startedAt": "2025-01-23T11:35:53.631082795Z",
"finishedAt": "2025-01-23T11:35:53.659037689Z", "finishedAt": "2025-01-23T11:35:53.659037689Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 19, "uid": 19,
@@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.006903297S", "duration": "PT0.006903297S",
"startedAt": "2025-01-20T11:50:52.874106134Z", "startedAt": "2025-01-20T11:50:52.874106134Z",
"finishedAt": "2025-01-20T11:50:52.881009431Z", "finishedAt": "2025-01-20T11:50:52.881009431Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 18, "uid": 18,
@@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000481257S", "duration": "PT0.000481257S",
"startedAt": "2025-01-20T11:48:04.92820416Z", "startedAt": "2025-01-20T11:48:04.92820416Z",
"finishedAt": "2025-01-20T11:48:04.928685417Z", "finishedAt": "2025-01-20T11:48:04.928685417Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 17, "uid": 17,
@@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000407005S", "duration": "PT0.000407005S",
"startedAt": "2025-01-20T11:47:53.509403957Z", "startedAt": "2025-01-20T11:47:53.509403957Z",
"finishedAt": "2025-01-20T11:47:53.509810962Z", "finishedAt": "2025-01-20T11:47:53.509810962Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 16, "uid": 16,
@@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000403716S", "duration": "PT0.000403716S",
"startedAt": "2025-01-20T11:47:48.430653005Z", "startedAt": "2025-01-20T11:47:48.430653005Z",
"finishedAt": "2025-01-20T11:47:48.431056721Z", "finishedAt": "2025-01-20T11:47:48.431056721Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 15, "uid": 15,
@@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000417016S", "duration": "PT0.000417016S",
"startedAt": "2025-01-20T11:47:42.429678617Z", "startedAt": "2025-01-20T11:47:42.429678617Z",
"finishedAt": "2025-01-20T11:47:42.430095633Z", "finishedAt": "2025-01-20T11:47:42.430095633Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 14, "uid": 14,
@@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT12.086284842S", "duration": "PT12.086284842S",
"startedAt": "2025-01-20T11:47:03.092181576Z", "startedAt": "2025-01-20T11:47:03.092181576Z",
"finishedAt": "2025-01-20T11:47:15.178466418Z", "finishedAt": "2025-01-20T11:47:15.178466418Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 13, "uid": 13,
@@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.011506614S", "duration": "PT0.011506614S",
"startedAt": "2025-01-16T17:18:43.29334923Z", "startedAt": "2025-01-16T17:18:43.29334923Z",
"finishedAt": "2025-01-16T17:18:43.304855844Z", "finishedAt": "2025-01-16T17:18:43.304855844Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 12, "uid": 12,
@@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007640163S", "duration": "PT0.007640163S",
"startedAt": "2025-01-16T17:02:52.539749853Z", "startedAt": "2025-01-16T17:02:52.539749853Z",
"finishedAt": "2025-01-16T17:02:52.547390016Z", "finishedAt": "2025-01-16T17:02:52.547390016Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 11, "uid": 11,
@@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007307840S", "duration": "PT0.007307840S",
"startedAt": "2025-01-16T17:01:14.112756687Z", "startedAt": "2025-01-16T17:01:14.112756687Z",
"finishedAt": "2025-01-16T17:01:14.120064527Z", "finishedAt": "2025-01-16T17:01:14.120064527Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 10, "uid": 10,
@@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007391353S", "duration": "PT0.007391353S",
"startedAt": "2025-01-16T17:00:29.201180268Z", "startedAt": "2025-01-16T17:00:29.201180268Z",
"finishedAt": "2025-01-16T17:00:29.208571621Z", "finishedAt": "2025-01-16T17:00:29.208571621Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 9, "uid": 9,
@@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007445825S", "duration": "PT0.007445825S",
"startedAt": "2025-01-16T17:00:15.77629445Z", "startedAt": "2025-01-16T17:00:15.77629445Z",
"finishedAt": "2025-01-16T17:00:15.783740275Z", "finishedAt": "2025-01-16T17:00:15.783740275Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 8, "uid": 8,
@@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.012020083S", "duration": "PT0.012020083S",
"startedAt": "2025-01-16T16:59:42.744086671Z", "startedAt": "2025-01-16T16:59:42.744086671Z",
"finishedAt": "2025-01-16T16:59:42.756106754Z", "finishedAt": "2025-01-16T16:59:42.756106754Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 7, "uid": 7,
@@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007440092S", "duration": "PT0.007440092S",
"startedAt": "2025-01-16T16:58:41.2155771Z", "startedAt": "2025-01-16T16:58:41.2155771Z",
"finishedAt": "2025-01-16T16:58:41.223017192Z", "finishedAt": "2025-01-16T16:58:41.223017192Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 6, "uid": 6,
@@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007565161S", "duration": "PT0.007565161S",
"startedAt": "2025-01-16T16:54:51.940332781Z", "startedAt": "2025-01-16T16:54:51.940332781Z",
"finishedAt": "2025-01-16T16:54:51.947897942Z", "finishedAt": "2025-01-16T16:54:51.947897942Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 5, "uid": 5,
@@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.016307263S", "duration": "PT0.016307263S",
"startedAt": "2025-01-16T16:53:19.913351957Z", "startedAt": "2025-01-16T16:53:19.913351957Z",
"finishedAt": "2025-01-16T16:53:19.92965922Z", "finishedAt": "2025-01-16T16:53:19.92965922Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 23, "total": 23,

42
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null, "progress": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
@@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task"
}, },
{ {
"uid": 23, "uid": 23,
@@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.004146631S", "duration": "PT0.004146631S",
"startedAt": "2025-01-23T11:38:57.012591321Z", "startedAt": "2025-01-23T11:38:57.012591321Z",
"finishedAt": "2025-01-23T11:38:57.016737952Z", "finishedAt": "2025-01-23T11:38:57.016737952Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 22, "uid": 22,
@@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.102738497S", "duration": "PT0.102738497S",
"startedAt": "2025-01-23T11:36:22.551906856Z", "startedAt": "2025-01-23T11:36:22.551906856Z",
"finishedAt": "2025-01-23T11:36:22.654645353Z", "finishedAt": "2025-01-23T11:36:22.654645353Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 21, "uid": 21,
@@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.005108474S", "duration": "PT0.005108474S",
"startedAt": "2025-01-23T11:36:04.132670526Z", "startedAt": "2025-01-23T11:36:04.132670526Z",
"finishedAt": "2025-01-23T11:36:04.137779Z", "finishedAt": "2025-01-23T11:36:04.137779Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 20, "uid": 20,
@@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.027954894S", "duration": "PT0.027954894S",
"startedAt": "2025-01-23T11:35:53.631082795Z", "startedAt": "2025-01-23T11:35:53.631082795Z",
"finishedAt": "2025-01-23T11:35:53.659037689Z", "finishedAt": "2025-01-23T11:35:53.659037689Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 19, "uid": 19,
@@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.006903297S", "duration": "PT0.006903297S",
"startedAt": "2025-01-20T11:50:52.874106134Z", "startedAt": "2025-01-20T11:50:52.874106134Z",
"finishedAt": "2025-01-20T11:50:52.881009431Z", "finishedAt": "2025-01-20T11:50:52.881009431Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 18, "uid": 18,
@@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000481257S", "duration": "PT0.000481257S",
"startedAt": "2025-01-20T11:48:04.92820416Z", "startedAt": "2025-01-20T11:48:04.92820416Z",
"finishedAt": "2025-01-20T11:48:04.928685417Z", "finishedAt": "2025-01-20T11:48:04.928685417Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 17, "uid": 17,
@@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000407005S", "duration": "PT0.000407005S",
"startedAt": "2025-01-20T11:47:53.509403957Z", "startedAt": "2025-01-20T11:47:53.509403957Z",
"finishedAt": "2025-01-20T11:47:53.509810962Z", "finishedAt": "2025-01-20T11:47:53.509810962Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 16, "uid": 16,
@@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000403716S", "duration": "PT0.000403716S",
"startedAt": "2025-01-20T11:47:48.430653005Z", "startedAt": "2025-01-20T11:47:48.430653005Z",
"finishedAt": "2025-01-20T11:47:48.431056721Z", "finishedAt": "2025-01-20T11:47:48.431056721Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 15, "uid": 15,
@@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000417016S", "duration": "PT0.000417016S",
"startedAt": "2025-01-20T11:47:42.429678617Z", "startedAt": "2025-01-20T11:47:42.429678617Z",
"finishedAt": "2025-01-20T11:47:42.430095633Z", "finishedAt": "2025-01-20T11:47:42.430095633Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 14, "uid": 14,
@@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT12.086284842S", "duration": "PT12.086284842S",
"startedAt": "2025-01-20T11:47:03.092181576Z", "startedAt": "2025-01-20T11:47:03.092181576Z",
"finishedAt": "2025-01-20T11:47:15.178466418Z", "finishedAt": "2025-01-20T11:47:15.178466418Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 13, "uid": 13,
@@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.011506614S", "duration": "PT0.011506614S",
"startedAt": "2025-01-16T17:18:43.29334923Z", "startedAt": "2025-01-16T17:18:43.29334923Z",
"finishedAt": "2025-01-16T17:18:43.304855844Z", "finishedAt": "2025-01-16T17:18:43.304855844Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 12, "uid": 12,
@@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007640163S", "duration": "PT0.007640163S",
"startedAt": "2025-01-16T17:02:52.539749853Z", "startedAt": "2025-01-16T17:02:52.539749853Z",
"finishedAt": "2025-01-16T17:02:52.547390016Z", "finishedAt": "2025-01-16T17:02:52.547390016Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 11, "uid": 11,
@@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007307840S", "duration": "PT0.007307840S",
"startedAt": "2025-01-16T17:01:14.112756687Z", "startedAt": "2025-01-16T17:01:14.112756687Z",
"finishedAt": "2025-01-16T17:01:14.120064527Z", "finishedAt": "2025-01-16T17:01:14.120064527Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 10, "uid": 10,
@@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007391353S", "duration": "PT0.007391353S",
"startedAt": "2025-01-16T17:00:29.201180268Z", "startedAt": "2025-01-16T17:00:29.201180268Z",
"finishedAt": "2025-01-16T17:00:29.208571621Z", "finishedAt": "2025-01-16T17:00:29.208571621Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 9, "uid": 9,
@@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007445825S", "duration": "PT0.007445825S",
"startedAt": "2025-01-16T17:00:15.77629445Z", "startedAt": "2025-01-16T17:00:15.77629445Z",
"finishedAt": "2025-01-16T17:00:15.783740275Z", "finishedAt": "2025-01-16T17:00:15.783740275Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 8, "uid": 8,
@@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.012020083S", "duration": "PT0.012020083S",
"startedAt": "2025-01-16T16:59:42.744086671Z", "startedAt": "2025-01-16T16:59:42.744086671Z",
"finishedAt": "2025-01-16T16:59:42.756106754Z", "finishedAt": "2025-01-16T16:59:42.756106754Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 7, "uid": 7,
@@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007440092S", "duration": "PT0.007440092S",
"startedAt": "2025-01-16T16:58:41.2155771Z", "startedAt": "2025-01-16T16:58:41.2155771Z",
"finishedAt": "2025-01-16T16:58:41.223017192Z", "finishedAt": "2025-01-16T16:58:41.223017192Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 6, "uid": 6,
@@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007565161S", "duration": "PT0.007565161S",
"startedAt": "2025-01-16T16:54:51.940332781Z", "startedAt": "2025-01-16T16:54:51.940332781Z",
"finishedAt": "2025-01-16T16:54:51.947897942Z", "finishedAt": "2025-01-16T16:54:51.947897942Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 5, "uid": 5,
@@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.016307263S", "duration": "PT0.016307263S",
"startedAt": "2025-01-16T16:53:19.913351957Z", "startedAt": "2025-01-16T16:53:19.913351957Z",
"finishedAt": "2025-01-16T16:53:19.92965922Z", "finishedAt": "2025-01-16T16:53:19.92965922Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 23, "total": 23,

42
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null, "progress": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
@@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task"
}, },
{ {
"uid": 23, "uid": 23,
@@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.004146631S", "duration": "PT0.004146631S",
"startedAt": "2025-01-23T11:38:57.012591321Z", "startedAt": "2025-01-23T11:38:57.012591321Z",
"finishedAt": "2025-01-23T11:38:57.016737952Z", "finishedAt": "2025-01-23T11:38:57.016737952Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 22, "uid": 22,
@@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.102738497S", "duration": "PT0.102738497S",
"startedAt": "2025-01-23T11:36:22.551906856Z", "startedAt": "2025-01-23T11:36:22.551906856Z",
"finishedAt": "2025-01-23T11:36:22.654645353Z", "finishedAt": "2025-01-23T11:36:22.654645353Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 21, "uid": 21,
@@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.005108474S", "duration": "PT0.005108474S",
"startedAt": "2025-01-23T11:36:04.132670526Z", "startedAt": "2025-01-23T11:36:04.132670526Z",
"finishedAt": "2025-01-23T11:36:04.137779Z", "finishedAt": "2025-01-23T11:36:04.137779Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 20, "uid": 20,
@@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.027954894S", "duration": "PT0.027954894S",
"startedAt": "2025-01-23T11:35:53.631082795Z", "startedAt": "2025-01-23T11:35:53.631082795Z",
"finishedAt": "2025-01-23T11:35:53.659037689Z", "finishedAt": "2025-01-23T11:35:53.659037689Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 19, "uid": 19,
@@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.006903297S", "duration": "PT0.006903297S",
"startedAt": "2025-01-20T11:50:52.874106134Z", "startedAt": "2025-01-20T11:50:52.874106134Z",
"finishedAt": "2025-01-20T11:50:52.881009431Z", "finishedAt": "2025-01-20T11:50:52.881009431Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 18, "uid": 18,
@@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000481257S", "duration": "PT0.000481257S",
"startedAt": "2025-01-20T11:48:04.92820416Z", "startedAt": "2025-01-20T11:48:04.92820416Z",
"finishedAt": "2025-01-20T11:48:04.928685417Z", "finishedAt": "2025-01-20T11:48:04.928685417Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 17, "uid": 17,
@@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000407005S", "duration": "PT0.000407005S",
"startedAt": "2025-01-20T11:47:53.509403957Z", "startedAt": "2025-01-20T11:47:53.509403957Z",
"finishedAt": "2025-01-20T11:47:53.509810962Z", "finishedAt": "2025-01-20T11:47:53.509810962Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 16, "uid": 16,
@@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000403716S", "duration": "PT0.000403716S",
"startedAt": "2025-01-20T11:47:48.430653005Z", "startedAt": "2025-01-20T11:47:48.430653005Z",
"finishedAt": "2025-01-20T11:47:48.431056721Z", "finishedAt": "2025-01-20T11:47:48.431056721Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 15, "uid": 15,
@@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000417016S", "duration": "PT0.000417016S",
"startedAt": "2025-01-20T11:47:42.429678617Z", "startedAt": "2025-01-20T11:47:42.429678617Z",
"finishedAt": "2025-01-20T11:47:42.430095633Z", "finishedAt": "2025-01-20T11:47:42.430095633Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 14, "uid": 14,
@@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT12.086284842S", "duration": "PT12.086284842S",
"startedAt": "2025-01-20T11:47:03.092181576Z", "startedAt": "2025-01-20T11:47:03.092181576Z",
"finishedAt": "2025-01-20T11:47:15.178466418Z", "finishedAt": "2025-01-20T11:47:15.178466418Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 13, "uid": 13,
@@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.011506614S", "duration": "PT0.011506614S",
"startedAt": "2025-01-16T17:18:43.29334923Z", "startedAt": "2025-01-16T17:18:43.29334923Z",
"finishedAt": "2025-01-16T17:18:43.304855844Z", "finishedAt": "2025-01-16T17:18:43.304855844Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 12, "uid": 12,
@@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007640163S", "duration": "PT0.007640163S",
"startedAt": "2025-01-16T17:02:52.539749853Z", "startedAt": "2025-01-16T17:02:52.539749853Z",
"finishedAt": "2025-01-16T17:02:52.547390016Z", "finishedAt": "2025-01-16T17:02:52.547390016Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 11, "uid": 11,
@@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007307840S", "duration": "PT0.007307840S",
"startedAt": "2025-01-16T17:01:14.112756687Z", "startedAt": "2025-01-16T17:01:14.112756687Z",
"finishedAt": "2025-01-16T17:01:14.120064527Z", "finishedAt": "2025-01-16T17:01:14.120064527Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 10, "uid": 10,
@@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007391353S", "duration": "PT0.007391353S",
"startedAt": "2025-01-16T17:00:29.201180268Z", "startedAt": "2025-01-16T17:00:29.201180268Z",
"finishedAt": "2025-01-16T17:00:29.208571621Z", "finishedAt": "2025-01-16T17:00:29.208571621Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 9, "uid": 9,
@@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007445825S", "duration": "PT0.007445825S",
"startedAt": "2025-01-16T17:00:15.77629445Z", "startedAt": "2025-01-16T17:00:15.77629445Z",
"finishedAt": "2025-01-16T17:00:15.783740275Z", "finishedAt": "2025-01-16T17:00:15.783740275Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 8, "uid": 8,
@@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.012020083S", "duration": "PT0.012020083S",
"startedAt": "2025-01-16T16:59:42.744086671Z", "startedAt": "2025-01-16T16:59:42.744086671Z",
"finishedAt": "2025-01-16T16:59:42.756106754Z", "finishedAt": "2025-01-16T16:59:42.756106754Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 7, "uid": 7,
@@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007440092S", "duration": "PT0.007440092S",
"startedAt": "2025-01-16T16:58:41.2155771Z", "startedAt": "2025-01-16T16:58:41.2155771Z",
"finishedAt": "2025-01-16T16:58:41.223017192Z", "finishedAt": "2025-01-16T16:58:41.223017192Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 6, "uid": 6,
@@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007565161S", "duration": "PT0.007565161S",
"startedAt": "2025-01-16T16:54:51.940332781Z", "startedAt": "2025-01-16T16:54:51.940332781Z",
"finishedAt": "2025-01-16T16:54:51.947897942Z", "finishedAt": "2025-01-16T16:54:51.947897942Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 5, "uid": 5,
@@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.016307263S", "duration": "PT0.016307263S",
"startedAt": "2025-01-16T16:53:19.913351957Z", "startedAt": "2025-01-16T16:53:19.913351957Z",
"finishedAt": "2025-01-16T16:53:19.92965922Z", "finishedAt": "2025-01-16T16:53:19.92965922Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 23, "total": 23,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_batchUids_equal_10.snap
View File

@@ -29,7 +29,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 1, "total": 1,

4
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 0, "uid": 0,
@@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.111055654S", "duration": "PT0.111055654S",
"startedAt": "2025-01-16T16:45:16.020248085Z", "startedAt": "2025-01-16T16:45:16.020248085Z",
"finishedAt": "2025-01-16T16:45:16.131303739Z", "finishedAt": "2025-01-16T16:45:16.131303739Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 2, "total": 2,

4
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 0, "uid": 0,
@@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.111055654S", "duration": "PT0.111055654S",
"startedAt": "2025-01-16T16:45:16.020248085Z", "startedAt": "2025-01-16T16:45:16.020248085Z",
"finishedAt": "2025-01-16T16:45:16.131303739Z", "finishedAt": "2025-01-16T16:45:16.131303739Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 2, "total": 2,

4
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -25,7 +25,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 0, "uid": 0,
@@ -49,7 +49,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.111055654S", "duration": "PT0.111055654S",
"startedAt": "2025-01-16T16:45:16.020248085Z", "startedAt": "2025-01-16T16:45:16.020248085Z",
"finishedAt": "2025-01-16T16:45:16.131303739Z", "finishedAt": "2025-01-16T16:45:16.131303739Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 2, "total": 2,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_canceledBy_equal_19.snap
View File

@@ -30,7 +30,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 1, "total": 1,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_statuses_equal_canceled.snap
View File

@@ -30,7 +30,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 1, "total": 1,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_uids_equal_10.snap
View File

@@ -29,7 +29,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 1, "total": 1,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null, "canceledBy": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"error": null, "error": null,
"duration": "[duration]", "duration": "[duration]",

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null, "canceledBy": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"error": null, "error": null,
"duration": "[duration]", "duration": "[duration]",

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null, "canceledBy": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"error": null, "error": null,
"duration": "[duration]", "duration": "[duration]",

52
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
View File

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"progress": null, "progress": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"stats": { "stats": {
"totalNbTasks": 1, "totalNbTasks": 1,
@@ -24,7 +24,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "[duration]", "duration": "[duration]",
"startedAt": "[date]", "startedAt": "[date]",
"finishedAt": "[date]", "finishedAt": "[date]",
"batcherStoppedBecause": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task" "batchCreationComplete": "a batch of tasks of type `upgradeDatabase` cannot be batched with any other type of task"
}, },
{ {
"uid": 23, "uid": 23,
@@ -47,7 +47,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.004146631S", "duration": "PT0.004146631S",
"startedAt": "2025-01-23T11:38:57.012591321Z", "startedAt": "2025-01-23T11:38:57.012591321Z",
"finishedAt": "2025-01-23T11:38:57.016737952Z", "finishedAt": "2025-01-23T11:38:57.016737952Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 22, "uid": 22,
@@ -71,7 +71,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.102738497S", "duration": "PT0.102738497S",
"startedAt": "2025-01-23T11:36:22.551906856Z", "startedAt": "2025-01-23T11:36:22.551906856Z",
"finishedAt": "2025-01-23T11:36:22.654645353Z", "finishedAt": "2025-01-23T11:36:22.654645353Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 21, "uid": 21,
@@ -95,7 +95,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.005108474S", "duration": "PT0.005108474S",
"startedAt": "2025-01-23T11:36:04.132670526Z", "startedAt": "2025-01-23T11:36:04.132670526Z",
"finishedAt": "2025-01-23T11:36:04.137779Z", "finishedAt": "2025-01-23T11:36:04.137779Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 20, "uid": 20,
@@ -119,7 +119,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.027954894S", "duration": "PT0.027954894S",
"startedAt": "2025-01-23T11:35:53.631082795Z", "startedAt": "2025-01-23T11:35:53.631082795Z",
"finishedAt": "2025-01-23T11:35:53.659037689Z", "finishedAt": "2025-01-23T11:35:53.659037689Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 19, "uid": 19,
@@ -142,7 +142,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.006903297S", "duration": "PT0.006903297S",
"startedAt": "2025-01-20T11:50:52.874106134Z", "startedAt": "2025-01-20T11:50:52.874106134Z",
"finishedAt": "2025-01-20T11:50:52.881009431Z", "finishedAt": "2025-01-20T11:50:52.881009431Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 18, "uid": 18,
@@ -171,7 +171,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000481257S", "duration": "PT0.000481257S",
"startedAt": "2025-01-20T11:48:04.92820416Z", "startedAt": "2025-01-20T11:48:04.92820416Z",
"finishedAt": "2025-01-20T11:48:04.928685417Z", "finishedAt": "2025-01-20T11:48:04.928685417Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 17, "uid": 17,
@@ -194,7 +194,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000407005S", "duration": "PT0.000407005S",
"startedAt": "2025-01-20T11:47:53.509403957Z", "startedAt": "2025-01-20T11:47:53.509403957Z",
"finishedAt": "2025-01-20T11:47:53.509810962Z", "finishedAt": "2025-01-20T11:47:53.509810962Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 16, "uid": 16,
@@ -217,7 +217,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000403716S", "duration": "PT0.000403716S",
"startedAt": "2025-01-20T11:47:48.430653005Z", "startedAt": "2025-01-20T11:47:48.430653005Z",
"finishedAt": "2025-01-20T11:47:48.431056721Z", "finishedAt": "2025-01-20T11:47:48.431056721Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 15, "uid": 15,
@@ -240,7 +240,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.000417016S", "duration": "PT0.000417016S",
"startedAt": "2025-01-20T11:47:42.429678617Z", "startedAt": "2025-01-20T11:47:42.429678617Z",
"finishedAt": "2025-01-20T11:47:42.430095633Z", "finishedAt": "2025-01-20T11:47:42.430095633Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 14, "uid": 14,
@@ -264,7 +264,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT12.086284842S", "duration": "PT12.086284842S",
"startedAt": "2025-01-20T11:47:03.092181576Z", "startedAt": "2025-01-20T11:47:03.092181576Z",
"finishedAt": "2025-01-20T11:47:15.178466418Z", "finishedAt": "2025-01-20T11:47:15.178466418Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 13, "uid": 13,
@@ -296,7 +296,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.011506614S", "duration": "PT0.011506614S",
"startedAt": "2025-01-16T17:18:43.29334923Z", "startedAt": "2025-01-16T17:18:43.29334923Z",
"finishedAt": "2025-01-16T17:18:43.304855844Z", "finishedAt": "2025-01-16T17:18:43.304855844Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 12, "uid": 12,
@@ -324,7 +324,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007640163S", "duration": "PT0.007640163S",
"startedAt": "2025-01-16T17:02:52.539749853Z", "startedAt": "2025-01-16T17:02:52.539749853Z",
"finishedAt": "2025-01-16T17:02:52.547390016Z", "finishedAt": "2025-01-16T17:02:52.547390016Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 11, "uid": 11,
@@ -347,7 +347,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007307840S", "duration": "PT0.007307840S",
"startedAt": "2025-01-16T17:01:14.112756687Z", "startedAt": "2025-01-16T17:01:14.112756687Z",
"finishedAt": "2025-01-16T17:01:14.120064527Z", "finishedAt": "2025-01-16T17:01:14.120064527Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 10, "uid": 10,
@@ -375,7 +375,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007391353S", "duration": "PT0.007391353S",
"startedAt": "2025-01-16T17:00:29.201180268Z", "startedAt": "2025-01-16T17:00:29.201180268Z",
"finishedAt": "2025-01-16T17:00:29.208571621Z", "finishedAt": "2025-01-16T17:00:29.208571621Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 9, "uid": 9,
@@ -403,7 +403,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007445825S", "duration": "PT0.007445825S",
"startedAt": "2025-01-16T17:00:15.77629445Z", "startedAt": "2025-01-16T17:00:15.77629445Z",
"finishedAt": "2025-01-16T17:00:15.783740275Z", "finishedAt": "2025-01-16T17:00:15.783740275Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 8, "uid": 8,
@@ -436,7 +436,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.012020083S", "duration": "PT0.012020083S",
"startedAt": "2025-01-16T16:59:42.744086671Z", "startedAt": "2025-01-16T16:59:42.744086671Z",
"finishedAt": "2025-01-16T16:59:42.756106754Z", "finishedAt": "2025-01-16T16:59:42.756106754Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 7, "uid": 7,
@@ -463,7 +463,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007440092S", "duration": "PT0.007440092S",
"startedAt": "2025-01-16T16:58:41.2155771Z", "startedAt": "2025-01-16T16:58:41.2155771Z",
"finishedAt": "2025-01-16T16:58:41.223017192Z", "finishedAt": "2025-01-16T16:58:41.223017192Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 6, "uid": 6,
@@ -490,7 +490,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007565161S", "duration": "PT0.007565161S",
"startedAt": "2025-01-16T16:54:51.940332781Z", "startedAt": "2025-01-16T16:54:51.940332781Z",
"finishedAt": "2025-01-16T16:54:51.947897942Z", "finishedAt": "2025-01-16T16:54:51.947897942Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 5, "uid": 5,
@@ -516,7 +516,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.016307263S", "duration": "PT0.016307263S",
"startedAt": "2025-01-16T16:53:19.913351957Z", "startedAt": "2025-01-16T16:53:19.913351957Z",
"finishedAt": "2025-01-16T16:53:19.92965922Z", "finishedAt": "2025-01-16T16:53:19.92965922Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 4, "uid": 4,
@@ -540,7 +540,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.087655941S", "duration": "PT0.087655941S",
"startedAt": "2025-01-16T16:52:32.631145531Z", "startedAt": "2025-01-16T16:52:32.631145531Z",
"finishedAt": "2025-01-16T16:52:32.718801472Z", "finishedAt": "2025-01-16T16:52:32.718801472Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 3, "uid": 3,
@@ -565,7 +565,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.007593573S", "duration": "PT0.007593573S",
"startedAt": "2025-01-16T16:47:53.677901409Z", "startedAt": "2025-01-16T16:47:53.677901409Z",
"finishedAt": "2025-01-16T16:47:53.685494982Z", "finishedAt": "2025-01-16T16:47:53.685494982Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 2, "uid": 2,
@@ -591,7 +591,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.017769760S", "duration": "PT0.017769760S",
"startedAt": "2025-01-16T16:47:41.211587682Z", "startedAt": "2025-01-16T16:47:41.211587682Z",
"finishedAt": "2025-01-16T16:47:41.229357442Z", "finishedAt": "2025-01-16T16:47:41.229357442Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 1, "uid": 1,
@@ -615,7 +615,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.066095506S", "duration": "PT0.066095506S",
"startedAt": "2025-01-16T16:47:10.217299609Z", "startedAt": "2025-01-16T16:47:10.217299609Z",
"finishedAt": "2025-01-16T16:47:10.283395115Z", "finishedAt": "2025-01-16T16:47:10.283395115Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
}, },
{ {
"uid": 0, "uid": 0,
@@ -639,7 +639,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"duration": "PT0.111055654S", "duration": "PT0.111055654S",
"startedAt": "2025-01-16T16:45:16.020248085Z", "startedAt": "2025-01-16T16:45:16.020248085Z",
"finishedAt": "2025-01-16T16:45:16.131303739Z", "finishedAt": "2025-01-16T16:45:16.131303739Z",
"batcherStoppedBecause": "unspecified" "batchCreationComplete": "unspecified"
} }
], ],
"total": 25, "total": 25,

2
crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap
View File

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
"canceledBy": null, "canceledBy": null,
"details": { "details": {
"upgradeFrom": "v1.12.0", "upgradeFrom": "v1.12.0",
"upgradeTo": "v1.14.0" "upgradeTo": "v1.15.0"
}, },
"error": null, "error": null,
"duration": "[duration]", "duration": "[duration]",

22
crates/meilitool/src/upgrade/mod.rs
View File

@@ -49,15 +49,10 @@ impl OfflineUpgrade {
const LAST_SUPPORTED_UPGRADE_TO_VERSION: &str = "1.12.7"; const LAST_SUPPORTED_UPGRADE_TO_VERSION: &str = "1.12.7";
let upgrade_list = [ let upgrade_list = [
( (v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, 1, 10, 0),
v1_9_to_v1_10 as fn(&Path, u32, u32, u32) -> Result<(), anyhow::Error>, (v1_10_to_v1_11, 1, 11, 0),
"1", (v1_11_to_v1_12, 1, 12, 0),
"10", (v1_12_to_v1_12_3, 1, 12, 3),
"0",
),
(v1_10_to_v1_11, "1", "11", "0"),
(v1_11_to_v1_12, "1", "12", "0"),
(v1_12_to_v1_12_3, "1", "12", "3"),
]; ];
let no_upgrade: usize = upgrade_list.len(); let no_upgrade: usize = upgrade_list.len();
@@ -95,13 +90,8 @@ impl OfflineUpgrade {
if start_at == no_upgrade { if start_at == no_upgrade {
println!("No upgrade operation to perform, writing VERSION file"); println!("No upgrade operation to perform, writing VERSION file");
create_version_file( create_version_file(&self.db_path, target_major, target_minor, target_patch)
&self.db_path, .context("while writing VERSION file after the upgrade")?;
&target_major.to_string(),
&target_minor.to_string(),
&target_patch.to_string(),
)
.context("while writing VERSION file after the upgrade")?;
println!("Success"); println!("Success");
return Ok(()); return Ok(());
} }

13
crates/milli/src/constants.rs
View File

@@ -1,6 +1,13 @@
pub static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR"); pub const VERSION_MAJOR: u32 = parse_u32(env!("CARGO_PKG_VERSION_MAJOR"));
pub static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR"); pub const VERSION_MINOR: u32 = parse_u32(env!("CARGO_PKG_VERSION_MINOR"));
pub static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH"); pub const VERSION_PATCH: u32 = parse_u32(env!("CARGO_PKG_VERSION_PATCH"));
const fn parse_u32(s: &str) -> u32 {
match u32::from_str_radix(s, 10) {
Ok(version) => version,
Err(_) => panic!("could not parse as u32"),
}
}
pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors"; pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors";
pub const RESERVED_GEO_FIELD_NAME: &str = "_geo"; pub const RESERVED_GEO_FIELD_NAME: &str = "_geo";

50
crates/milli/src/disabled_typos_terms.rs Normal file
View File

@@ -0,0 +1,50 @@
use heed::{
types::{SerdeJson, Str},
RoTxn, RwTxn,
};
use serde::{Deserialize, Serialize};
use crate::{index::main_key, Index};
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "camelCase")]
pub struct DisabledTyposTerms {
pub disable_on_numbers: bool,
}
impl Index {
pub fn disabled_typos_terms(&self, txn: &RoTxn<'_>) -> heed::Result<DisabledTyposTerms> {
self.main
.remap_types::<Str, SerdeJson<DisabledTyposTerms>>()
.get(txn, main_key::DISABLED_TYPOS_TERMS)
.map(|option| option.unwrap_or_default())
}
pub(crate) fn put_disabled_typos_terms(
&self,
txn: &mut RwTxn<'_>,
disabled_typos_terms: &DisabledTyposTerms,
) -> heed::Result<()> {
self.main.remap_types::<Str, SerdeJson<DisabledTyposTerms>>().put(
txn,
main_key::DISABLED_TYPOS_TERMS,
disabled_typos_terms,
)?;
Ok(())
}
pub(crate) fn delete_disabled_typos_terms(&self, txn: &mut RwTxn<'_>) -> heed::Result<()> {
self.main
.remap_types::<Str, SerdeJson<DisabledTyposTerms>>()
.delete(txn, main_key::DISABLED_TYPOS_TERMS)?;
Ok(())
}
}
impl DisabledTyposTerms {
pub fn is_exact(&self, word: &str) -> bool {
// If disable_on_numbers is true, we disable the word if it contains only numbers or punctuation
self.disable_on_numbers && word.chars().all(|c| c.is_numeric() || c.is_ascii_punctuation())
}
}

1506
crates/milli/src/index.rs
View File

File diff suppressed because it is too large Load Diff

1
crates/milli/src/lib.rs
View File

@@ -12,6 +12,7 @@ mod asc_desc;
mod attribute_patterns; mod attribute_patterns;
mod criterion; mod criterion;
pub mod database_stats; pub mod database_stats;
pub mod disabled_typos_terms;
mod error; mod error;
mod external_documents_ids; mod external_documents_ids;
pub mod facet; pub mod facet;

99
crates/milli/src/search/facet/filter.rs
View File

@@ -1,10 +1,11 @@
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::fmt::{Debug, Display}; use std::fmt::{Debug, Display};
use std::ops::Bound::{self, Excluded, Included}; use std::ops::Bound::{self, Excluded, Included, Unbounded};
use either::Either; use either::Either;
pub use filter_parser::{Condition, Error as FPError, FilterCondition, Token}; pub use filter_parser::{Condition, Error as FPError, FilterCondition, Token};
use heed::types::LazyDecode; use heed::types::LazyDecode;
use heed::BytesEncode;
use memchr::memmem::Finder; use memchr::memmem::Finder;
use roaring::{MultiOps, RoaringBitmap}; use roaring::{MultiOps, RoaringBitmap};
use serde_json::Value; use serde_json::Value;
@@ -14,7 +15,7 @@ use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::error::{Error, UserError}; use crate::error::{Error, UserError};
use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features};
use crate::heed_codec::facet::{ use crate::heed_codec::facet::{
FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, OrderedF64Codec, FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec,
}; };
use crate::index::db_name::FACET_ID_STRING_DOCIDS; use crate::index::db_name::FACET_ID_STRING_DOCIDS;
use crate::{ use crate::{
@@ -271,7 +272,7 @@ impl<'a> Filter<'a> {
// as the facets values are all in the same database and prefixed by the // as the facets values are all in the same database and prefixed by the
// field id and the level. // field id and the level.
let (left, right) = match operator { let (number_bounds, (left_str, right_str)) = match operator {
// return an error if the filter is not allowed for this field // return an error if the filter is not allowed for this field
Condition::GreaterThan(_) Condition::GreaterThan(_)
| Condition::GreaterThanOrEqual(_) | Condition::GreaterThanOrEqual(_)
@@ -305,17 +306,37 @@ impl<'a> Filter<'a> {
)); ));
} }
Condition::GreaterThan(val) => { Condition::GreaterThan(val) => {
(Excluded(val.parse_finite_float()?), Included(f64::MAX)) let number = val.parse_finite_float().ok();
let number_bounds = number.map(|number| (Excluded(number), Included(f64::MAX)));
let str_bounds = (Excluded(val.value()), Unbounded);
(number_bounds, str_bounds)
} }
Condition::GreaterThanOrEqual(val) => { Condition::GreaterThanOrEqual(val) => {
(Included(val.parse_finite_float()?), Included(f64::MAX)) let number = val.parse_finite_float().ok();
let number_bounds = number.map(|number| (Included(number), Included(f64::MAX)));
let str_bounds = (Included(val.value()), Unbounded);
(number_bounds, str_bounds)
}
Condition::LowerThan(val) => {
let number = val.parse_finite_float().ok();
let number_bounds = number.map(|number| (Included(f64::MIN), Excluded(number)));
let str_bounds = (Unbounded, Excluded(val.value()));
(number_bounds, str_bounds)
} }
Condition::LowerThan(val) => (Included(f64::MIN), Excluded(val.parse_finite_float()?)),
Condition::LowerThanOrEqual(val) => { Condition::LowerThanOrEqual(val) => {
(Included(f64::MIN), Included(val.parse_finite_float()?)) let number = val.parse_finite_float().ok();
let number_bounds = number.map(|number| (Included(f64::MIN), Included(number)));
let str_bounds = (Unbounded, Included(val.value()));
(number_bounds, str_bounds)
} }
Condition::Between { from, to } => { Condition::Between { from, to } => {
(Included(from.parse_finite_float()?), Included(to.parse_finite_float()?)) let from_number = from.parse_finite_float().ok();
let to_number = to.parse_finite_float().ok();
let number_bounds =
from_number.zip(to_number).map(|(from, to)| (Included(from), Included(to)));
let str_bounds = (Included(from.value()), Included(to.value()));
(number_bounds, str_bounds)
} }
Condition::Null => { Condition::Null => {
let is_null = index.null_faceted_documents_ids(rtxn, field_id)?; let is_null = index.null_faceted_documents_ids(rtxn, field_id)?;
@@ -415,29 +436,47 @@ impl<'a> Filter<'a> {
}; };
let mut output = RoaringBitmap::new(); let mut output = RoaringBitmap::new();
Self::explore_facet_number_levels(
if let Some((left_number, right_number)) = number_bounds {
Self::explore_facet_levels(
rtxn,
numbers_db,
field_id,
&left_number,
&right_number,
universe,
&mut output,
)?;
}
Self::explore_facet_levels(
rtxn, rtxn,
numbers_db, strings_db,
field_id, field_id,
left, &left_str,
right, &right_str,
universe, universe,
&mut output, &mut output,
)?; )?;
Ok(output) Ok(output)
} }
/// Aggregates the documents ids that are part of the specified range automatically /// Aggregates the documents ids that are part of the specified range automatically
/// going deeper through the levels. /// going deeper through the levels.
fn explore_facet_number_levels( fn explore_facet_levels<'data, BoundCodec>(
rtxn: &heed::RoTxn<'_>, rtxn: &'data heed::RoTxn<'data>,
db: heed::Database<FacetGroupKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>, db: heed::Database<FacetGroupKeyCodec<BoundCodec>, FacetGroupValueCodec>,
field_id: FieldId, field_id: FieldId,
left: Bound<f64>, left: &'data Bound<<BoundCodec as heed::BytesEncode<'data>>::EItem>,
right: Bound<f64>, right: &'data Bound<<BoundCodec as heed::BytesEncode<'data>>::EItem>,
universe: Option<&RoaringBitmap>, universe: Option<&RoaringBitmap>,
output: &mut RoaringBitmap, output: &mut RoaringBitmap,
) -> Result<()> { ) -> Result<()>
where
BoundCodec: for<'b> BytesEncode<'b>,
for<'b> <BoundCodec as BytesEncode<'b>>::EItem: Sized + PartialOrd,
{
match (left, right) { match (left, right) {
// lower TO upper when lower > upper must return no result // lower TO upper when lower > upper must return no result
(Included(l), Included(r)) if l > r => return Ok(()), (Included(l), Included(r)) if l > r => return Ok(()),
@@ -446,8 +485,8 @@ impl<'a> Filter<'a> {
(Excluded(l), Included(r)) if l >= r => return Ok(()), (Excluded(l), Included(r)) if l >= r => return Ok(()),
(_, _) => (), (_, _) => (),
} }
facet_range_search::find_docids_of_facet_within_bounds::<OrderedF64Codec>( facet_range_search::find_docids_of_facet_within_bounds::<BoundCodec>(
rtxn, db, field_id, &left, &right, universe, output, rtxn, db, field_id, left, right, universe, output,
)?; )?;
Ok(()) Ok(())
@@ -1249,28 +1288,24 @@ mod tests {
let result = filter.evaluate(&rtxn, &index).unwrap(); let result = filter.evaluate(&rtxn, &index).unwrap();
assert!(result.contains(0)); assert!(result.contains(0));
let filter = Filter::from_str("price < inf").unwrap().unwrap(); let filter = Filter::from_str("price < inf").unwrap().unwrap();
assert!(matches!( let result = filter.evaluate(&rtxn, &index).unwrap();
filter.evaluate(&rtxn, &index), // this is allowed due to filters with strings
Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_))) assert!(result.contains(1));
));
let filter = Filter::from_str("price = NaN").unwrap().unwrap(); let filter = Filter::from_str("price = NaN").unwrap().unwrap();
let result = filter.evaluate(&rtxn, &index).unwrap(); let result = filter.evaluate(&rtxn, &index).unwrap();
assert!(result.is_empty()); assert!(result.is_empty());
let filter = Filter::from_str("price < NaN").unwrap().unwrap(); let filter = Filter::from_str("price < NaN").unwrap().unwrap();
assert!(matches!( let result = filter.evaluate(&rtxn, &index).unwrap();
filter.evaluate(&rtxn, &index), assert!(result.contains(1));
Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_)))
));
let filter = Filter::from_str("price = infinity").unwrap().unwrap(); let filter = Filter::from_str("price = infinity").unwrap().unwrap();
let result = filter.evaluate(&rtxn, &index).unwrap(); let result = filter.evaluate(&rtxn, &index).unwrap();
assert!(result.contains(2)); assert!(result.contains(2));
let filter = Filter::from_str("price < infinity").unwrap().unwrap(); let filter = Filter::from_str("price < infinity").unwrap().unwrap();
assert!(matches!( let result = filter.evaluate(&rtxn, &index).unwrap();
filter.evaluate(&rtxn, &index), assert!(result.contains(0));
Err(crate::Error::UserError(crate::error::UserError::InvalidFilter(_))) assert!(result.contains(1));
));
} }
#[test] #[test]

8
crates/milli/src/search/mod.rs
View File

@@ -8,7 +8,7 @@ use roaring::bitmap::RoaringBitmap;
pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET}; pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET};
pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords}; pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords};
use self::new::{execute_vector_search, PartialSearchResult}; use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats};
use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features};
use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::score_details::{ScoreDetails, ScoringStrategy};
use crate::vector::Embedder; use crate::vector::Embedder;
@@ -269,6 +269,12 @@ impl<'a> Search<'a> {
)?, )?,
}; };
if let Some(VectorStoreStats { total_time, total_queries, total_results }) =
ctx.vector_store_stats
{
tracing::debug!("Vector store stats: total_time={total_time:.02?}, total_queries={total_queries}, total_results={total_results}");
}
// consume context and located_query_terms to build MatchingWords. // consume context and located_query_terms to build MatchingWords.
let matching_words = match located_query_terms { let matching_words = match located_query_terms {
Some(located_query_terms) => MatchingWords::new(ctx, located_query_terms), Some(located_query_terms) => MatchingWords::new(ctx, located_query_terms),

23
crates/milli/src/search/new/mod.rs
View File

@@ -22,6 +22,8 @@ mod vector_sort;
mod tests; mod tests;
use std::collections::HashSet; use std::collections::HashSet;
use std::ops::AddAssign;
use std::time::Duration;
use bucket_sort::{bucket_sort, BucketSortOutput}; use bucket_sort::{bucket_sort, BucketSortOutput};
use charabia::{Language, TokenizerBuilder}; use charabia::{Language, TokenizerBuilder};
@@ -72,6 +74,7 @@ pub struct SearchContext<'ctx> {
pub phrase_docids: PhraseDocIdsCache, pub phrase_docids: PhraseDocIdsCache,
pub restricted_fids: Option<RestrictedFids>, pub restricted_fids: Option<RestrictedFids>,
pub prefix_search: PrefixSearch, pub prefix_search: PrefixSearch,
pub vector_store_stats: Option<VectorStoreStats>,
} }
impl<'ctx> SearchContext<'ctx> { impl<'ctx> SearchContext<'ctx> {
@@ -101,6 +104,7 @@ impl<'ctx> SearchContext<'ctx> {
phrase_docids: <_>::default(), phrase_docids: <_>::default(),
restricted_fids: None, restricted_fids: None,
prefix_search, prefix_search,
vector_store_stats: None,
}) })
} }
@@ -166,6 +170,25 @@ impl<'ctx> SearchContext<'ctx> {
} }
} }
#[derive(Debug, Default)]
pub struct VectorStoreStats {
/// The total time spent on vector search.
pub total_time: Duration,
/// The number of searches performed.
pub total_queries: usize,
/// The number of nearest neighbors found.
pub total_results: usize,
}
impl AddAssign for VectorStoreStats {
fn add_assign(&mut self, other: Self) {
let Self { total_time, total_queries, total_results } = self;
*total_time += other.total_time;
*total_queries += other.total_queries;
*total_results += other.total_results;
}
}
#[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)] #[derive(Clone, Copy, PartialEq, PartialOrd, Ord, Eq)]
pub enum Word { pub enum Word {
Original(Interned<String>), Original(Interned<String>),

8
crates/milli/src/search/new/vector_sort.rs
View File

@@ -1,8 +1,10 @@
use std::iter::FromIterator; use std::iter::FromIterator;
use std::time::Instant;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait};
use super::VectorStoreStats;
use crate::score_details::{self, ScoreDetails}; use crate::score_details::{self, ScoreDetails};
use crate::vector::{ArroyWrapper, DistributionShift, Embedder}; use crate::vector::{ArroyWrapper, DistributionShift, Embedder};
use crate::{DocumentId, Result, SearchContext, SearchLogger}; use crate::{DocumentId, Result, SearchContext, SearchLogger};
@@ -53,9 +55,15 @@ impl<Q: RankingRuleQueryTrait> VectorSort<Q> {
) -> Result<()> { ) -> Result<()> {
let target = &self.target; let target = &self.target;
let before = Instant::now();
let reader = ArroyWrapper::new(ctx.index.vector_arroy, self.embedder_index, self.quantized); let reader = ArroyWrapper::new(ctx.index.vector_arroy, self.embedder_index, self.quantized);
let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?; let results = reader.nns_by_vector(ctx.txn, target, self.limit, Some(vector_candidates))?;
self.cached_sorted_docids = results.into_iter(); self.cached_sorted_docids = results.into_iter();
*ctx.vector_store_stats.get_or_insert_default() += VectorStoreStats {
total_time: before.elapsed(),
total_queries: 1,
total_results: self.cached_sorted_docids.len(),
};
Ok(()) Ok(())
} }

4
crates/milli/src/snapshots/index.rs/bug_3007/geo_faceted_documents_ids.snap
View File

@@ -1,4 +0,0 @@
---
source: milli/src/index.rs
---
[0, ]

4
crates/milli/src/snapshots/index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap
View File

@@ -1,4 +0,0 @@
---
source: milli/src/index.rs
---
[]

4
crates/milli/src/snapshots/test_index.rs/bug_3007/geo_faceted_documents_ids.snap Normal file
View File

@@ -0,0 +1,4 @@
---
source: crates/milli/src/test_index.rs
---
[0, ]

4
crates/milli/src/snapshots/test_index.rs/unexpected_extra_fields_in_geo_field/geo_faceted_documents_ids.snap Normal file
View File

@@ -0,0 +1,4 @@
---
source: crates/milli/src/test_index.rs
---
[]

1399
crates/milli/src/test_index.rs Normal file
View File

File diff suppressed because it is too large Load Diff

6
crates/milli/src/update/index_documents/extract/extract_word_docids.rs
View File

@@ -127,7 +127,8 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
// merge all deletions // merge all deletions
let obkv = KvReaderDelAdd::from_slice(value); let obkv = KvReaderDelAdd::from_slice(value);
if let Some(value) = obkv.get(DelAdd::Deletion) { if let Some(value) = obkv.get(DelAdd::Deletion) {
let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid); let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid)
|| settings_diff.old.disabled_typos_terms.is_exact(w);
buffer.clear(); buffer.clear();
let mut obkv = KvWriterDelAdd::new(&mut buffer); let mut obkv = KvWriterDelAdd::new(&mut buffer);
obkv.insert(DelAdd::Deletion, value)?; obkv.insert(DelAdd::Deletion, value)?;
@@ -139,7 +140,8 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
} }
// merge all additions // merge all additions
if let Some(value) = obkv.get(DelAdd::Addition) { if let Some(value) = obkv.get(DelAdd::Addition) {
let add_in_exact = settings_diff.new.exact_attributes.contains(&fid); let add_in_exact = settings_diff.new.exact_attributes.contains(&fid)
|| settings_diff.new.disabled_typos_terms.is_exact(w);
buffer.clear(); buffer.clear();
let mut obkv = KvWriterDelAdd::new(&mut buffer); let mut obkv = KvWriterDelAdd::new(&mut buffer);
obkv.insert(DelAdd::Addition, value)?; obkv.insert(DelAdd::Addition, value)?;

3
crates/milli/src/update/index_documents/typed_chunk.rs
View File

@@ -273,14 +273,11 @@ pub(crate) fn write_typed_chunk_into_index(
unreachable!(); unreachable!();
}; };
let clonable_word_docids = unsafe { as_cloneable_grenad(&word_docids_reader) }?; let clonable_word_docids = unsafe { as_cloneable_grenad(&word_docids_reader) }?;
let clonable_exact_word_docids =
unsafe { as_cloneable_grenad(&exact_word_docids_reader) }?;
word_docids_builder.push(word_docids_reader.into_cursor()?); word_docids_builder.push(word_docids_reader.into_cursor()?);
exact_word_docids_builder.push(exact_word_docids_reader.into_cursor()?); exact_word_docids_builder.push(exact_word_docids_reader.into_cursor()?);
word_fid_docids_builder.push(word_fid_docids_reader.into_cursor()?); word_fid_docids_builder.push(word_fid_docids_reader.into_cursor()?);
fst_merger_builder.push(clonable_word_docids.into_cursor()?); fst_merger_builder.push(clonable_word_docids.into_cursor()?);
fst_merger_builder.push(clonable_exact_word_docids.into_cursor()?);
} }
let word_docids_merger = word_docids_builder.build(); let word_docids_merger = word_docids_builder.build();

15
crates/milli/src/update/new/extract/searchable/extract_word_docids.rs
View File

@@ -319,8 +319,11 @@ impl WordDocidsExtractors {
let doc_alloc = &context.doc_alloc; let doc_alloc = &context.doc_alloc;
let exact_attributes = index.exact_attributes(rtxn)?; let exact_attributes = index.exact_attributes(rtxn)?;
let is_exact_attribute = let disabled_typos_terms = index.disabled_typos_terms(rtxn)?;
|fname: &str| exact_attributes.iter().any(|attr| contained_in(fname, attr)); let is_exact = |fname: &str, word: &str| {
exact_attributes.iter().any(|attr| contained_in(fname, attr))
|| disabled_typos_terms.is_exact(word)
};
match document_change { match document_change {
DocumentChange::Deletion(inner) => { DocumentChange::Deletion(inner) => {
let mut token_fn = |fname: &str, fid, pos, word: &str| { let mut token_fn = |fname: &str, fid, pos, word: &str| {
@@ -328,7 +331,7 @@ impl WordDocidsExtractors {
fid, fid,
pos, pos,
word, word,
is_exact_attribute(fname), is_exact(fname, word),
inner.docid(), inner.docid(),
doc_alloc, doc_alloc,
) )
@@ -356,7 +359,7 @@ impl WordDocidsExtractors {
fid, fid,
pos, pos,
word, word,
is_exact_attribute(fname), is_exact(fname, word),
inner.docid(), inner.docid(),
doc_alloc, doc_alloc,
) )
@@ -372,7 +375,7 @@ impl WordDocidsExtractors {
fid, fid,
pos, pos,
word, word,
is_exact_attribute(fname), is_exact(fname, word),
inner.docid(), inner.docid(),
doc_alloc, doc_alloc,
) )
@@ -389,7 +392,7 @@ impl WordDocidsExtractors {
fid, fid,
pos, pos,
word, word,
is_exact_attribute(fname), is_exact(fname, word),
inner.docid(), inner.docid(),
doc_alloc, doc_alloc,
) )

1
crates/milli/src/update/new/indexer/mod.rs
View File

@@ -9,6 +9,7 @@ pub use document_operation::{DocumentOperation, PayloadStats};
use hashbrown::HashMap; use hashbrown::HashMap;
use heed::RwTxn; use heed::RwTxn;
pub use partial_dump::PartialDump; pub use partial_dump::PartialDump;
pub use post_processing::recompute_word_fst_from_word_docids_database;
pub use update_by_function::UpdateByFunction; pub use update_by_function::UpdateByFunction;
pub use write::ChannelCongestion; pub use write::ChannelCongestion;
use write::{build_vectors, update_index, write_to_db}; use write::{build_vectors, update_index, write_to_db};

14
crates/milli/src/update/new/indexer/post_processing.rs
View File

@@ -131,6 +131,20 @@ fn compute_word_fst(
} }
} }
pub fn recompute_word_fst_from_word_docids_database(index: &Index, wtxn: &mut RwTxn) -> Result<()> {
let fst = fst::Set::default().map_data(std::borrow::Cow::Owned)?;
let mut word_fst_builder = WordFstBuilder::new(&fst)?;
let words = index.word_docids.iter(wtxn)?.remap_data_type::<DecodeIgnore>();
for res in words {
let (word, _) = res?;
word_fst_builder.register_word(DelAdd::Addition, word.as_ref())?;
}
let (word_fst_mmap, _) = word_fst_builder.build(index, wtxn)?;
index.main.remap_types::<Str, Bytes>().put(wtxn, WORDS_FST_KEY, &word_fst_mmap)?;
Ok(())
}
#[tracing::instrument(level = "trace", skip_all, target = "indexing::facet_search")] #[tracing::instrument(level = "trace", skip_all, target = "indexing::facet_search")]
fn compute_facet_search_database( fn compute_facet_search_database(
index: &Index, index: &Index,

35
crates/milli/src/update/settings.rs
View File

@@ -17,6 +17,7 @@ use super::IndexerConfig;
use crate::attribute_patterns::PatternMatch; use crate::attribute_patterns::PatternMatch;
use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::criterion::Criterion; use crate::criterion::Criterion;
use crate::disabled_typos_terms::DisabledTyposTerms;
use crate::error::UserError; use crate::error::UserError;
use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder}; use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder};
use crate::filterable_attributes_rules::match_faceted_field; use crate::filterable_attributes_rules::match_faceted_field;
@@ -169,6 +170,7 @@ pub struct Settings<'a, 't, 'i> {
synonyms: Setting<BTreeMap<String, Vec<String>>>, synonyms: Setting<BTreeMap<String, Vec<String>>>,
primary_key: Setting<String>, primary_key: Setting<String>,
authorize_typos: Setting<bool>, authorize_typos: Setting<bool>,
disable_on_numbers: Setting<bool>,
min_word_len_two_typos: Setting<u8>, min_word_len_two_typos: Setting<u8>,
min_word_len_one_typo: Setting<u8>, min_word_len_one_typo: Setting<u8>,
exact_words: Setting<BTreeSet<String>>, exact_words: Setting<BTreeSet<String>>,
@@ -207,6 +209,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
synonyms: Setting::NotSet, synonyms: Setting::NotSet,
primary_key: Setting::NotSet, primary_key: Setting::NotSet,
authorize_typos: Setting::NotSet, authorize_typos: Setting::NotSet,
disable_on_numbers: Setting::NotSet,
exact_words: Setting::NotSet, exact_words: Setting::NotSet,
min_word_len_two_typos: Setting::NotSet, min_word_len_two_typos: Setting::NotSet,
min_word_len_one_typo: Setting::NotSet, min_word_len_one_typo: Setting::NotSet,
@@ -354,6 +357,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
self.min_word_len_one_typo = Setting::Reset; self.min_word_len_one_typo = Setting::Reset;
} }
pub fn set_disable_on_numbers(&mut self, disable_on_numbers: bool) {
self.disable_on_numbers = Setting::Set(disable_on_numbers);
}
pub fn reset_disable_on_numbers(&mut self) {
self.disable_on_numbers = Setting::Reset;
}
pub fn set_exact_words(&mut self, words: BTreeSet<String>) { pub fn set_exact_words(&mut self, words: BTreeSet<String>) {
self.exact_words = Setting::Set(words); self.exact_words = Setting::Set(words);
} }
@@ -866,6 +877,24 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
Ok(()) Ok(())
} }
fn update_disabled_typos_terms(&mut self) -> Result<()> {
let mut disabled_typos_terms = self.index.disabled_typos_terms(self.wtxn)?;
match self.disable_on_numbers {
Setting::Set(disable_on_numbers) => {
disabled_typos_terms.disable_on_numbers = disable_on_numbers;
}
Setting::Reset => {
self.index.delete_disabled_typos_terms(self.wtxn)?;
disabled_typos_terms.disable_on_numbers =
DisabledTyposTerms::default().disable_on_numbers;
}
Setting::NotSet => (),
}
self.index.put_disabled_typos_terms(self.wtxn, &disabled_typos_terms)?;
Ok(())
}
fn update_exact_words(&mut self) -> Result<()> { fn update_exact_words(&mut self) -> Result<()> {
match self.exact_words { match self.exact_words {
Setting::Set(ref mut words) => { Setting::Set(ref mut words) => {
@@ -1246,6 +1275,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
self.update_prefix_search()?; self.update_prefix_search()?;
self.update_facet_search()?; self.update_facet_search()?;
self.update_localized_attributes_rules()?; self.update_localized_attributes_rules()?;
self.update_disabled_typos_terms()?;
let embedding_config_updates = self.update_embedding_configs()?; let embedding_config_updates = self.update_embedding_configs()?;
@@ -1327,6 +1357,7 @@ impl InnerIndexSettingsDiff {
|| old_settings.prefix_search != new_settings.prefix_search || old_settings.prefix_search != new_settings.prefix_search
|| old_settings.localized_attributes_rules || old_settings.localized_attributes_rules
!= new_settings.localized_attributes_rules != new_settings.localized_attributes_rules
|| old_settings.disabled_typos_terms != new_settings.disabled_typos_terms
}; };
let cache_exact_attributes = old_settings.exact_attributes != new_settings.exact_attributes; let cache_exact_attributes = old_settings.exact_attributes != new_settings.exact_attributes;
@@ -1526,6 +1557,7 @@ pub(crate) struct InnerIndexSettings {
pub user_defined_searchable_attributes: Option<Vec<String>>, pub user_defined_searchable_attributes: Option<Vec<String>>,
pub sortable_fields: HashSet<String>, pub sortable_fields: HashSet<String>,
pub exact_attributes: HashSet<FieldId>, pub exact_attributes: HashSet<FieldId>,
pub disabled_typos_terms: DisabledTyposTerms,
pub proximity_precision: ProximityPrecision, pub proximity_precision: ProximityPrecision,
pub embedding_configs: EmbeddingConfigs, pub embedding_configs: EmbeddingConfigs,
pub geo_fields_ids: Option<(FieldId, FieldId)>, pub geo_fields_ids: Option<(FieldId, FieldId)>,
@@ -1574,7 +1606,7 @@ impl InnerIndexSettings {
.map(|fields| fields.into_iter().map(|f| f.to_string()).collect()); .map(|fields| fields.into_iter().map(|f| f.to_string()).collect());
let builder = MetadataBuilder::from_index(index, rtxn)?; let builder = MetadataBuilder::from_index(index, rtxn)?;
let fields_ids_map = FieldIdMapWithMetadata::new(fields_ids_map, builder); let fields_ids_map = FieldIdMapWithMetadata::new(fields_ids_map, builder);
let disabled_typos_terms = index.disabled_typos_terms(rtxn)?;
Ok(Self { Ok(Self {
stop_words, stop_words,
allowed_separators, allowed_separators,
@@ -1592,6 +1624,7 @@ impl InnerIndexSettings {
geo_fields_ids, geo_fields_ids,
prefix_search, prefix_search,
facet_search, facet_search,
disabled_typos_terms,
}) })
} }

2
crates/milli/src/update/test_settings.rs
View File

@@ -896,6 +896,7 @@ fn test_correct_settings_init() {
localized_attributes_rules, localized_attributes_rules,
prefix_search, prefix_search,
facet_search, facet_search,
disable_on_numbers,
} = settings; } = settings;
assert!(matches!(searchable_fields, Setting::NotSet)); assert!(matches!(searchable_fields, Setting::NotSet));
assert!(matches!(displayed_fields, Setting::NotSet)); assert!(matches!(displayed_fields, Setting::NotSet));
@@ -923,6 +924,7 @@ fn test_correct_settings_init() {
assert!(matches!(localized_attributes_rules, Setting::NotSet)); assert!(matches!(localized_attributes_rules, Setting::NotSet));
assert!(matches!(prefix_search, Setting::NotSet)); assert!(matches!(prefix_search, Setting::NotSet));
assert!(matches!(facet_search, Setting::NotSet)); assert!(matches!(facet_search, Setting::NotSet));
assert!(matches!(disable_on_numbers, Setting::NotSet));
}) })
.unwrap(); .unwrap();
} }

41
crates/milli/src/update/upgrade/mod.rs
View File

@@ -1,12 +1,14 @@
mod v1_12; mod v1_12;
mod v1_13; mod v1_13;
mod v1_14; mod v1_14;
mod v1_15;
use heed::RwTxn; use heed::RwTxn;
use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3};
use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13};
use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_14::Latest_V1_13_To_Latest_V1_14;
use v1_15::Latest_V1_14_To_Latest_V1_15;
use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
use crate::progress::{Progress, VariableNameStep}; use crate::progress::{Progress, VariableNameStep};
use crate::{Index, InternalError, Result}; use crate::{Index, InternalError, Result};
@@ -23,12 +25,16 @@ trait UpgradeIndex {
} }
/// Return true if the cached stats of the index must be regenerated /// Return true if the cached stats of the index must be regenerated
pub fn upgrade( pub fn upgrade<MSP>(
wtxn: &mut RwTxn, wtxn: &mut RwTxn,
index: &Index, index: &Index,
db_version: (u32, u32, u32), db_version: (u32, u32, u32),
must_stop_processing: MSP,
progress: Progress, progress: Progress,
) -> Result<bool> { ) -> Result<bool>
where
MSP: Fn() -> bool + Sync,
{
let from = index.get_version(wtxn)?.unwrap_or(db_version); let from = index.get_version(wtxn)?.unwrap_or(db_version);
let upgrade_functions: &[&dyn UpgradeIndex] = &[ let upgrade_functions: &[&dyn UpgradeIndex] = &[
&V1_12_To_V1_12_3 {}, &V1_12_To_V1_12_3 {},
@@ -36,6 +42,10 @@ pub fn upgrade(
&V1_13_0_To_V1_13_1 {}, &V1_13_0_To_V1_13_1 {},
&V1_13_1_To_Latest_V1_13 {}, &V1_13_1_To_Latest_V1_13 {},
&Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_13_To_Latest_V1_14 {},
&Latest_V1_14_To_Latest_V1_15 {},
// This is the last upgrade function, it will be called when the index is up to date.
// any other upgrade function should be added before this one.
&ToCurrentNoOp {},
]; ];
let start = match from { let start = match from {
@@ -43,8 +53,9 @@ pub fn upgrade(
(1, 12, 3..) => 1, (1, 12, 3..) => 1,
(1, 13, 0) => 2, (1, 13, 0) => 2,
(1, 13, _) => 4, (1, 13, _) => 4,
(1, 14, _) => 5,
// We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other.
(1, 14, _) => 4, (1, 15, _) => 6,
(major, minor, patch) => { (major, minor, patch) => {
return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into())
} }
@@ -56,6 +67,9 @@ pub fn upgrade(
let mut current_version = from; let mut current_version = from;
let mut regenerate_stats = false; let mut regenerate_stats = false;
for (i, upgrade) in upgrade_path.iter().enumerate() { for (i, upgrade) in upgrade_path.iter().enumerate() {
if (must_stop_processing)() {
return Err(crate::Error::InternalError(InternalError::AbortedIndexation));
}
let target = upgrade.target_version(); let target = upgrade.target_version();
progress.update_progress(VariableNameStep::<UpgradeVersion>::new( progress.update_progress(VariableNameStep::<UpgradeVersion>::new(
format!( format!(
@@ -77,3 +91,22 @@ pub fn upgrade(
Ok(regenerate_stats) Ok(regenerate_stats)
} }
#[allow(non_camel_case_types)]
struct ToCurrentNoOp {}
impl UpgradeIndex for ToCurrentNoOp {
fn upgrade(
&self,
_wtxn: &mut RwTxn,
_index: &Index,
_original: (u32, u32, u32),
_progress: Progress,
) -> Result<bool> {
Ok(false)
}
fn target_version(&self) -> (u32, u32, u32) {
(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
}
}

7
crates/milli/src/update/upgrade/v1_13.rs
View File

@@ -1,7 +1,6 @@
use heed::RwTxn; use heed::RwTxn;
use super::UpgradeIndex; use super::UpgradeIndex;
use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
use crate::database_stats::DatabaseStats; use crate::database_stats::DatabaseStats;
use crate::progress::Progress; use crate::progress::Progress;
use crate::{make_enum_progress, Index, Result}; use crate::{make_enum_progress, Index, Result};
@@ -51,10 +50,6 @@ impl UpgradeIndex for V1_13_1_To_Latest_V1_13 {
} }
fn target_version(&self) -> (u32, u32, u32) { fn target_version(&self) -> (u32, u32, u32) {
( (1, 13, 3)
VERSION_MAJOR.parse().unwrap(),
VERSION_MINOR.parse().unwrap(),
VERSION_PATCH.parse().unwrap(),
)
} }
} }

35
crates/milli/src/update/upgrade/v1_15.rs Normal file
View File

@@ -0,0 +1,35 @@
use heed::RwTxn;
use super::UpgradeIndex;
use crate::progress::Progress;
use crate::update::new::indexer::recompute_word_fst_from_word_docids_database;
use crate::{make_enum_progress, Index, Result};
#[allow(non_camel_case_types)]
pub(super) struct Latest_V1_14_To_Latest_V1_15();
impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 {
fn upgrade(
&self,
wtxn: &mut RwTxn,
index: &Index,
_original: (u32, u32, u32),
progress: Progress,
) -> Result<bool> {
// Recompute the word FST from the word docids database.
make_enum_progress! {
enum TypoTolerance {
RecomputeWordFst,
}
};
progress.update_progress(TypoTolerance::RecomputeWordFst);
recompute_word_fst_from_word_docids_database(index, wtxn)?;
Ok(false)
}
fn target_version(&self) -> (u32, u32, u32) {
(1, 15, 0)
}
}