",
@@ -36,6 +36,12 @@ license = "MIT"
[profile.release]
codegen-units = 1
+# We now compile heed without the NDEBUG define for better performance.
+# However, we still enable debug assertions for a better detection of
+# disk corruption on the cloud or in OSS.
+[profile.release.package.heed]
+debug-assertions = true
+
[profile.dev.package.flate2]
opt-level = 3
@@ -43,6 +49,3 @@ opt-level = 3
opt-level = 3
[profile.dev.package.roaring]
opt-level = 3
-
-[patch.crates-io]
-roaring = { git = "https://github.com/RoaringBitmap/roaring-rs", branch = "clone-iter-slice" }
diff --git a/Dockerfile b/Dockerfile
index 04557df59..5a9a4691f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
# Compile
-FROM rust:1.79.0-alpine3.20 AS compiler
+FROM rust:1.85-alpine3.20 AS compiler
RUN apk add -q --no-cache build-base openssl-dev
diff --git a/LICENSE b/LICENSE
index 76a573977..686f24931 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2019-2024 Meili SAS
+Copyright (c) 2019-2025 Meili SAS
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index 4be92d439..d85942584 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,13 @@
- 
+ 
+
+
+
+
+ 
+
ā” A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow š
@@ -58,6 +64,7 @@ See the list of all our example apps in our [demos repository](https://github.co
- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
- **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **AI-ready:** works out of the box with [langchain](https://www.meilisearch.com/with/langchain) and the [model context protocol](https://github.com/meilisearch/meilisearch-mcp)
- **Easy to install, deploy, and maintain**
## š Documentation
diff --git a/assets/grafana-dashboard.json b/assets/grafana-dashboard.json
index 2cfa85a46..027afcbba 100644
--- a/assets/grafana-dashboard.json
+++ b/assets/grafana-dashboard.json
@@ -1403,6 +1403,104 @@
"title": "Number of tasks by indexes",
"type": "timeseries"
},
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 15,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "id": 29,
+ "interval": "5s",
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.1.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "builder",
+ "exemplar": true,
+ "expr": "meilisearch_task_queue_latency_seconds{instance=\"$instance\", job=\"$job\"}",
+ "interval": "",
+ "legendFormat": "{{value}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Task queue latency",
+ "type": "timeseries"
+ },
{
"collapsed": true,
"datasource": {
diff --git a/assets/ph-banner.png b/assets/ph-banner.png
new file mode 100644
index 000000000..399c7fd10
Binary files /dev/null and b/assets/ph-banner.png differ
diff --git a/bors.toml b/bors.toml
deleted file mode 100644
index 96e9ef65e..000000000
--- a/bors.toml
+++ /dev/null
@@ -1,11 +0,0 @@
-status = [
- 'Tests on ubuntu-20.04',
- 'Tests on macos-13',
- 'Tests on windows-2022',
- 'Run Clippy',
- 'Run Rustfmt',
- 'Run tests in debug',
-]
-pr_status = ['Milestone Check']
-# 3 hours timeout
-timeout-sec = 10800
diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml
index eec30ea3f..a2cddd554 100644
--- a/crates/benchmarks/Cargo.toml
+++ b/crates/benchmarks/Cargo.toml
@@ -11,27 +11,27 @@ edition.workspace = true
license.workspace = true
[dependencies]
-anyhow = "1.0.86"
+anyhow = "1.0.95"
bumpalo = "3.16.0"
-csv = "1.3.0"
+csv = "1.3.1"
memmap2 = "0.9.5"
milli = { path = "../milli" }
mimalloc = { version = "0.1.43", default-features = false }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
-tempfile = "3.14.0"
+serde_json = { version = "1.0.135", features = ["preserve_order"] }
+tempfile = "3.15.0"
[dev-dependencies]
criterion = { version = "0.5.1", features = ["html_reports"] }
rand = "0.8.5"
rand_chacha = "0.3.1"
-roaring = "0.10.6"
+roaring = "0.10.10"
[build-dependencies]
-anyhow = "1.0.86"
-bytes = "1.6.0"
+anyhow = "1.0.95"
+bytes = "1.9.0"
convert_case = "0.6.0"
-flate2 = "1.0.30"
-reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.35"
+reqwest = { version = "0.12.12", features = ["blocking", "rustls-tls"], default-features = false }
[features]
default = ["milli/all-tokenizations"]
diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs
index 2f33c3454..9199c3877 100644
--- a/crates/benchmarks/benches/indexing.rs
+++ b/crates/benchmarks/benches/indexing.rs
@@ -8,14 +8,16 @@ use bumpalo::Bump;
use criterion::{criterion_group, criterion_main, Criterion};
use milli::documents::PrimaryKey;
use milli::heed::{EnvOpenOptions, RwTxn};
+use milli::progress::Progress;
use milli::update::new::indexer;
-use milli::update::{IndexDocumentsMethod, IndexerConfig, Settings};
+use milli::update::{IndexerConfig, Settings};
use milli::vector::EmbeddingConfigs;
-use milli::Index;
+use milli::{FilterableAttributesRule, Index};
use rand::seq::SliceRandom;
use rand_chacha::rand_core::SeedableRng;
use roaring::RoaringBitmap;
+#[cfg(not(windows))]
#[global_allocator]
static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
@@ -33,10 +35,11 @@ fn setup_dir(path: impl AsRef) {
fn setup_index() -> Index {
let path = "benches.mmdb";
setup_dir(path);
- let mut options = EnvOpenOptions::new();
+ let options = EnvOpenOptions::new();
+ let mut options = options.read_txn_without_tls();
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
options.max_readers(100);
- Index::new(options, path).unwrap()
+ Index::new(options, path, true).unwrap()
}
fn setup_settings<'t>(
@@ -55,7 +58,8 @@ fn setup_settings<'t>(
let searchable_fields = searchable_fields.iter().map(|s| s.to_string()).collect();
builder.set_searchable_fields(searchable_fields);
- let filterable_fields = filterable_fields.iter().map(|s| s.to_string()).collect();
+ let filterable_fields =
+ filterable_fields.iter().map(|s| FilterableAttributesRule::Field(s.to_string())).collect();
builder.set_filterable_fields(filterable_fields);
let sortable_fields = sortable_fields.iter().map(|s| s.to_string()).collect();
@@ -136,10 +140,9 @@ fn indexing_songs_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -150,13 +153,14 @@ fn indexing_songs_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -164,7 +168,7 @@ fn indexing_songs_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -202,10 +206,9 @@ fn reindexing_songs_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -216,13 +219,14 @@ fn reindexing_songs_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -230,7 +234,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -246,10 +250,9 @@ fn reindexing_songs_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -260,13 +263,14 @@ fn reindexing_songs_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -274,7 +278,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -314,10 +318,9 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -328,13 +331,14 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -342,7 +346,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -390,10 +394,9 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_1_2, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -404,13 +407,14 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -418,7 +422,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -434,10 +438,9 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_3_4, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -448,13 +451,14 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -462,7 +466,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -474,10 +478,9 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_4_4, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -488,13 +491,14 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -502,7 +506,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -540,11 +544,10 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -555,13 +558,14 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -569,7 +573,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -607,10 +611,9 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -621,13 +624,14 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -635,7 +639,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -673,10 +677,9 @@ fn indexing_wiki(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -687,13 +690,14 @@ fn indexing_wiki(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -701,7 +705,7 @@ fn indexing_wiki(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -738,10 +742,9 @@ fn reindexing_wiki(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -752,13 +755,14 @@ fn reindexing_wiki(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -766,7 +770,7 @@ fn reindexing_wiki(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -782,10 +786,9 @@ fn reindexing_wiki(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -796,13 +799,14 @@ fn reindexing_wiki(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -810,7 +814,7 @@ fn reindexing_wiki(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -849,10 +853,9 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -863,13 +866,14 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -877,7 +881,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -924,11 +928,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_1_2, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -939,13 +942,14 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -953,7 +957,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -969,11 +973,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_3_4, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -984,13 +987,14 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -998,7 +1002,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1010,11 +1014,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_4_4, "csv");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -1025,13 +1028,14 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -1039,7 +1043,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1077,10 +1081,9 @@ fn indexing_movies_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::MOVIES, "json");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -1091,13 +1094,14 @@ fn indexing_movies_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -1105,7 +1109,7 @@ fn indexing_movies_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1142,10 +1146,9 @@ fn reindexing_movies_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::MOVIES, "json");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -1156,13 +1159,14 @@ fn reindexing_movies_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -1170,7 +1174,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1186,10 +1190,9 @@ fn reindexing_movies_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::MOVIES, "json");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -1200,13 +1203,14 @@ fn reindexing_movies_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -1214,7 +1218,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1253,10 +1257,9 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
let db_fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
let mut new_fields_ids_map = db_fields_ids_map.clone();
- let mut indexer =
- indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
+ let mut indexer = indexer::DocumentOperation::new();
let documents = utils::documents_from(datasets_paths::MOVIES, "json");
- indexer.add_documents(&documents).unwrap();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -1267,13 +1270,14 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -1281,7 +1285,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
@@ -1321,6 +1325,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec Index {
}
create_dir_all(conf.database_name).unwrap();
- let mut options = EnvOpenOptions::new();
+ let options = EnvOpenOptions::new();
+ let mut options = options.read_txn_without_tls();
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
options.max_readers(100);
- let index = Index::new(options, conf.database_name).unwrap();
+ let index = Index::new(options, conf.database_name, true).unwrap();
let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap();
@@ -98,8 +100,8 @@ pub fn base_setup(conf: &Conf) -> Index {
let mut new_fields_ids_map = db_fields_ids_map.clone();
let documents = documents_from(conf.dataset, conf.dataset_format);
- let mut indexer = indexer::DocumentOperation::new(IndexDocumentsMethod::ReplaceDocuments);
- indexer.add_documents(&documents).unwrap();
+ let mut indexer = indexer::DocumentOperation::new();
+ indexer.replace_documents(&documents).unwrap();
let indexer_alloc = Bump::new();
let (document_changes, _operation_stats, primary_key) = indexer
@@ -110,13 +112,14 @@ pub fn base_setup(conf: &Conf) -> Index {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -124,7 +127,7 @@ pub fn base_setup(conf: &Conf) -> Index {
&document_changes,
EmbeddingConfigs::default(),
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
diff --git a/crates/build-info/Cargo.toml b/crates/build-info/Cargo.toml
index c24dffe5c..f8ede756e 100644
--- a/crates/build-info/Cargo.toml
+++ b/crates/build-info/Cargo.toml
@@ -11,8 +11,8 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-time = { version = "0.3.36", features = ["parsing"] }
+time = { version = "0.3.37", features = ["parsing"] }
[build-dependencies]
-anyhow = "1.0.86"
-vergen-git2 = "1.0.0"
+anyhow = "1.0.95"
+vergen-git2 = "1.0.2"
diff --git a/crates/dump/Cargo.toml b/crates/dump/Cargo.toml
index f9d2a9a0b..5c427916c 100644
--- a/crates/dump/Cargo.toml
+++ b/crates/dump/Cargo.toml
@@ -11,21 +11,21 @@ readme.workspace = true
license.workspace = true
[dependencies]
-anyhow = "1.0.86"
-flate2 = "1.0.30"
-http = "1.1.0"
+anyhow = "1.0.95"
+flate2 = "1.0.35"
+http = "1.2.0"
meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.19.0"
-regex = "1.10.5"
-roaring = { version = "0.10.6", features = ["serde"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
-tar = "0.4.41"
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-tracing = "0.1.40"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
+once_cell = "1.20.2"
+regex = "1.11.1"
+roaring = { version = "0.10.10", features = ["serde"] }
+serde = { version = "1.0.217", features = ["derive"] }
+serde_json = { version = "1.0.135", features = ["preserve_order"] }
+tar = "0.4.43"
+tempfile = "3.15.0"
+thiserror = "2.0.9"
+time = { version = "0.3.37", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tracing = "0.1.41"
+uuid = { version = "1.11.0", features = ["serde", "v4"] }
[dev-dependencies]
big_s = "1.0.2"
diff --git a/crates/dump/README.md b/crates/dump/README.md
index 3537f188e..42d84ec80 100644
--- a/crates/dump/README.md
+++ b/crates/dump/README.md
@@ -10,8 +10,10 @@ dump
āāā instance-uid.uuid
āāā keys.jsonl
āāā metadata.json
-āāā tasks
- āāā update_files
- ā āāā [task_id].jsonl
+āāā tasks
+ā āāā update_files
+ā ā āāā [task_id].jsonl
+ā āāā queue.jsonl
+āāā batches
āāā queue.jsonl
-```
\ No newline at end of file
+```
diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs
index 8bed7f0d4..4e2d6ac2f 100644
--- a/crates/dump/src/lib.rs
+++ b/crates/dump/src/lib.rs
@@ -141,6 +141,9 @@ pub enum KindDump {
instance_uid: Option,
},
SnapshotCreation,
+ UpgradeDatabase {
+ from: (u32, u32, u32),
+ },
}
impl From for TaskDump {
@@ -210,6 +213,9 @@ impl From for KindDump {
KindDump::DumpCreation { keys, instance_uid }
}
KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
+ KindWithContent::UpgradeDatabase { from: version } => {
+ KindDump::UpgradeDatabase { from: version }
+ }
}
}
}
@@ -222,14 +228,16 @@ pub(crate) mod test {
use big_s::S;
use maplit::{btreemap, btreeset};
+ use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats};
use meilisearch_types::facet_values_sort::FacetValuesSort;
- use meilisearch_types::features::RuntimeTogglableFeatures;
+ use meilisearch_types::features::{Network, Remote, RuntimeTogglableFeatures};
use meilisearch_types::index_uid_pattern::IndexUidPattern;
use meilisearch_types::keys::{Action, Key};
- use meilisearch_types::milli;
use meilisearch_types::milli::update::Setting;
+ use meilisearch_types::milli::{self, FilterableAttributesRule};
use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
- use meilisearch_types::tasks::{Details, Status};
+ use meilisearch_types::task_view::DetailsView;
+ use meilisearch_types::tasks::{Details, Kind, Status};
use serde_json::{json, Map, Value};
use time::macros::datetime;
use uuid::Uuid;
@@ -271,7 +279,10 @@ pub(crate) mod test {
let settings = Settings {
displayed_attributes: Setting::Set(vec![S("race"), S("name")]).into(),
searchable_attributes: Setting::Set(vec![S("name"), S("race")]).into(),
- filterable_attributes: Setting::Set(btreeset! { S("race"), S("age") }),
+ filterable_attributes: Setting::Set(vec![
+ FilterableAttributesRule::Field(S("race")),
+ FilterableAttributesRule::Field(S("age")),
+ ]),
sortable_attributes: Setting::Set(btreeset! { S("age") }),
ranking_rules: Setting::NotSet,
stop_words: Setting::NotSet,
@@ -292,11 +303,39 @@ pub(crate) mod test {
embedders: Setting::NotSet,
search_cutoff_ms: Setting::NotSet,
localized_attributes: Setting::NotSet,
+ facet_search: Setting::NotSet,
+ prefix_search: Setting::NotSet,
_kind: std::marker::PhantomData,
};
settings.check()
}
+ pub fn create_test_batches() -> Vec {
+ vec![Batch {
+ uid: 0,
+ details: DetailsView {
+ received_documents: Some(12),
+ indexed_documents: Some(Some(10)),
+ ..DetailsView::default()
+ },
+ progress: None,
+ stats: BatchStats {
+ total_nb_tasks: 1,
+ status: maplit::btreemap! { Status::Succeeded => 1 },
+ types: maplit::btreemap! { Kind::DocumentAdditionOrUpdate => 1 },
+ index_uids: maplit::btreemap! { "doggo".to_string() => 1 },
+ progress_trace: Default::default(),
+ write_channel_congestion: None,
+ },
+ enqueued_at: Some(BatchEnqueuedAt {
+ earliest: datetime!(2022-11-11 0:00 UTC),
+ oldest: datetime!(2022-11-11 0:00 UTC),
+ }),
+ started_at: datetime!(2022-11-20 0:00 UTC),
+ finished_at: Some(datetime!(2022-11-21 0:00 UTC)),
+ }]
+ }
+
pub fn create_test_tasks() -> Vec<(TaskDump, Option>)> {
vec![
(
@@ -419,6 +458,15 @@ pub(crate) mod test {
index.flush().unwrap();
index.settings(&settings).unwrap();
+ // ========== pushing the batch queue
+ let batches = create_test_batches();
+
+ let mut batch_queue = dump.create_batches_queue().unwrap();
+ for batch in &batches {
+ batch_queue.push_batch(batch).unwrap();
+ }
+ batch_queue.flush().unwrap();
+
// ========== pushing the task queue
let tasks = create_test_tasks();
@@ -447,6 +495,10 @@ pub(crate) mod test {
dump.create_experimental_features(features).unwrap();
+ // ========== network
+ let network = create_test_network();
+ dump.create_network(network).unwrap();
+
// create the dump
let mut file = tempfile::tempfile().unwrap();
dump.persist_to(&mut file).unwrap();
@@ -456,7 +508,14 @@ pub(crate) mod test {
}
fn create_test_features() -> RuntimeTogglableFeatures {
- RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
+ RuntimeTogglableFeatures::default()
+ }
+
+ fn create_test_network() -> Network {
+ Network {
+ local: Some("myself".to_string()),
+ remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()) }},
+ }
}
#[test]
@@ -507,5 +566,9 @@ pub(crate) mod test {
// ==== checking the features
let expected = create_test_features();
assert_eq!(dump.features().unwrap().unwrap(), expected);
+
+ // ==== checking the network
+ let expected = create_test_network();
+ assert_eq!(&expected, dump.network().unwrap().unwrap());
}
}
diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs
index 785542cce..6b63e7c6b 100644
--- a/crates/dump/src/reader/compat/v5_to_v6.rs
+++ b/crates/dump/src/reader/compat/v5_to_v6.rs
@@ -196,6 +196,10 @@ impl CompatV5ToV6 {
pub fn features(&self) -> Result> {
Ok(None)
}
+
+ pub fn network(&self) -> Result> {
+ Ok(None)
+ }
}
pub enum CompatIndexV5ToV6 {
@@ -318,7 +322,16 @@ impl From> for v6::Settings {
v6::Settings {
displayed_attributes: v6::Setting::from(settings.displayed_attributes).into(),
searchable_attributes: v6::Setting::from(settings.searchable_attributes).into(),
- filterable_attributes: settings.filterable_attributes.into(),
+ filterable_attributes: match settings.filterable_attributes {
+ v5::settings::Setting::Set(filterable_attributes) => v6::Setting::Set(
+ filterable_attributes
+ .into_iter()
+ .map(v6::FilterableAttributesRule::Field)
+ .collect(),
+ ),
+ v5::settings::Setting::Reset => v6::Setting::Reset,
+ v5::settings::Setting::NotSet => v6::Setting::NotSet,
+ },
sortable_attributes: settings.sortable_attributes.into(),
ranking_rules: {
match settings.ranking_rules {
@@ -382,6 +395,8 @@ impl From> for v6::Settings {
embedders: v6::Setting::NotSet,
localized_attributes: v6::Setting::NotSet,
search_cutoff_ms: v6::Setting::NotSet,
+ facet_search: v6::Setting::NotSet,
+ prefix_search: v6::Setting::NotSet,
_kind: std::marker::PhantomData,
}
}
diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs
index f9660972a..2b4440ab7 100644
--- a/crates/dump/src/reader/mod.rs
+++ b/crates/dump/src/reader/mod.rs
@@ -23,6 +23,7 @@ mod v6;
pub type Document = serde_json::Map;
pub type UpdateFile = dyn Iterator- >;
+#[allow(clippy::large_enum_variant)]
pub enum DumpReader {
Current(V6Reader),
Compat(CompatV5ToV6),
@@ -101,6 +102,13 @@ impl DumpReader {
}
}
+ pub fn batches(&mut self) -> Result> + '_>> {
+ match self {
+ DumpReader::Current(current) => Ok(current.batches()),
+ DumpReader::Compat(_compat) => Ok(Box::new(std::iter::empty())),
+ }
+ }
+
pub fn keys(&mut self) -> Result> + '_>> {
match self {
DumpReader::Current(current) => Ok(current.keys()),
@@ -114,6 +122,13 @@ impl DumpReader {
DumpReader::Compat(compat) => compat.features(),
}
}
+
+ pub fn network(&self) -> Result> {
+ match self {
+ DumpReader::Current(current) => Ok(current.network()),
+ DumpReader::Compat(compat) => compat.network(),
+ }
+ }
}
impl From for DumpReader {
@@ -219,6 +234,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -327,10 +346,8 @@ pub(crate) mod test {
}
}
- assert_eq!(
- dump.features().unwrap().unwrap(),
- RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
- );
+ assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default());
+ assert_eq!(dump.network().unwrap(), None);
}
#[test]
@@ -342,6 +359,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -373,10 +394,28 @@ pub(crate) mod test {
assert_eq!(test.documents().unwrap().count(), 1);
- assert_eq!(
- dump.features().unwrap().unwrap(),
- RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
- );
+ assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default());
+ }
+
+ #[test]
+ fn import_dump_v6_network() {
+ let dump = File::open("tests/assets/v6-with-network.dump").unwrap();
+ let dump = DumpReader::open(dump).unwrap();
+
+ // top level infos
+ insta::assert_snapshot!(dump.date().unwrap(), @"2025-01-29 15:45:32.738676 +00:00:00");
+ insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+
+ // network
+
+ let network = dump.network().unwrap().unwrap();
+ insta::assert_snapshot!(network.local.as_ref().unwrap(), @"ms-0");
+ insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().url, @"http://localhost:7700");
+ insta::assert_snapshot!(network.remotes.get("ms-0").as_ref().unwrap().search_api_key.is_none(), @"true");
+ insta::assert_snapshot!(network.remotes.get("ms-1").as_ref().unwrap().url, @"http://localhost:7701");
+ insta::assert_snapshot!(network.remotes.get("ms-1").as_ref().unwrap().search_api_key.is_none(), @"true");
+ insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().url, @"http://ms-5679.example.meilisearch.io");
+ insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().search_api_key.as_ref().unwrap(), @"foo");
}
#[test]
@@ -388,6 +427,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -468,6 +511,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -545,6 +592,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
assert_eq!(dump.instance_uid().unwrap(), None);
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -638,6 +689,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
assert_eq!(dump.instance_uid().unwrap(), None);
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -731,6 +786,10 @@ pub(crate) mod test {
insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
assert_eq!(dump.instance_uid().unwrap(), None);
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
@@ -807,6 +866,10 @@ pub(crate) mod test {
assert_eq!(dump.date(), None);
assert_eq!(dump.instance_uid().unwrap(), None);
+ // batches didn't exists at the time
+ let batches = dump.batches().unwrap().collect::>>().unwrap();
+ meili_snap::snapshot!(meili_snap::json_string!(batches), @"[]");
+
// tasks
let tasks = dump.tasks().unwrap().collect::>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
diff --git a/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap
index 77694a629..3c770a6eb 100644
--- a/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap
+++ b/crates/dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap
@@ -1,5 +1,5 @@
---
-source: dump/src/reader/mod.rs
+source: crates/dump/src/reader/mod.rs
expression: vector_index.settings().unwrap()
---
{
@@ -49,6 +49,7 @@ expression: vector_index.settings().unwrap()
"source": "huggingFace",
"model": "BAAI/bge-base-en-v1.5",
"revision": "617ca489d9e86b49b8167676d8220688b99db36e",
+ "pooling": "forceMean",
"documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}"
}
},
diff --git a/crates/dump/src/reader/v4/meta.rs b/crates/dump/src/reader/v4/meta.rs
index 2daea68a4..9b26eba25 100644
--- a/crates/dump/src/reader/v4/meta.rs
+++ b/crates/dump/src/reader/v4/meta.rs
@@ -108,7 +108,7 @@ where
/// not supported on untagged enums.
struct StarOrVisitor(PhantomData);
- impl<'de, T, FE> Visitor<'de> for StarOrVisitor
+ impl Visitor<'_> for StarOrVisitor
where
T: FromStr,
FE: Display,
diff --git a/crates/dump/src/reader/v4/tasks.rs b/crates/dump/src/reader/v4/tasks.rs
index a701d837d..8ae3f77b1 100644
--- a/crates/dump/src/reader/v4/tasks.rs
+++ b/crates/dump/src/reader/v4/tasks.rs
@@ -99,7 +99,7 @@ impl Task {
/// Return true when a task is finished.
/// A task is finished when its last state is either `Succeeded` or `Failed`.
pub fn is_finished(&self) -> bool {
- self.events.last().map_or(false, |event| {
+ self.events.last().is_some_and(|event| {
matches!(event, TaskEvent::Succeded { .. } | TaskEvent::Failed { .. })
})
}
diff --git a/crates/dump/src/reader/v5/meta.rs b/crates/dump/src/reader/v5/meta.rs
index 2daea68a4..9b26eba25 100644
--- a/crates/dump/src/reader/v5/meta.rs
+++ b/crates/dump/src/reader/v5/meta.rs
@@ -108,7 +108,7 @@ where
/// not supported on untagged enums.
struct StarOrVisitor(PhantomData);
- impl<'de, T, FE> Visitor<'de> for StarOrVisitor
+ impl Visitor<'_> for StarOrVisitor
where
T: FromStr,
FE: Display,
diff --git a/crates/dump/src/reader/v5/tasks.rs b/crates/dump/src/reader/v5/tasks.rs
index 8dfb2d0b0..a7352bf0c 100644
--- a/crates/dump/src/reader/v5/tasks.rs
+++ b/crates/dump/src/reader/v5/tasks.rs
@@ -114,7 +114,7 @@ impl Task {
/// Return true when a task is finished.
/// A task is finished when its last state is either `Succeeded` or `Failed`.
pub fn is_finished(&self) -> bool {
- self.events.last().map_or(false, |event| {
+ self.events.last().is_some_and(|event| {
matches!(event, TaskEvent::Succeeded { .. } | TaskEvent::Failed { .. })
})
}
diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs
index 50b9751a2..0b4ba5bdd 100644
--- a/crates/dump/src/reader/v6/mod.rs
+++ b/crates/dump/src/reader/v6/mod.rs
@@ -3,6 +3,7 @@ use std::io::{BufRead, BufReader, ErrorKind};
use std::path::Path;
pub use meilisearch_types::milli;
+use meilisearch_types::milli::vector::hf::OverridePooling;
use tempfile::TempDir;
use time::OffsetDateTime;
use tracing::debug;
@@ -18,8 +19,10 @@ pub type Checked = meilisearch_types::settings::Checked;
pub type Unchecked = meilisearch_types::settings::Unchecked;
pub type Task = crate::TaskDump;
+pub type Batch = meilisearch_types::batches::Batch;
pub type Key = meilisearch_types::keys::Key;
pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
+pub type Network = meilisearch_types::features::Network;
// ===== Other types to clarify the code of the compat module
// everything related to the tasks
@@ -43,13 +46,17 @@ pub type ResponseError = meilisearch_types::error::ResponseError;
pub type Code = meilisearch_types::error::Code;
pub type RankingRuleView = meilisearch_types::settings::RankingRuleView;
+pub type FilterableAttributesRule = meilisearch_types::milli::FilterableAttributesRule;
+
pub struct V6Reader {
dump: TempDir,
instance_uid: Option,
metadata: Metadata,
tasks: BufReader,
+ batches: Option>,
keys: BufReader,
features: Option,
+ network: Option,
}
impl V6Reader {
@@ -77,13 +84,38 @@ impl V6Reader {
} else {
None
};
+ let batches = match File::open(dump.path().join("batches").join("queue.jsonl")) {
+ Ok(file) => Some(BufReader::new(file)),
+ // The batch file was only introduced during the v1.13, anything prior to that won't have batches
+ Err(err) if err.kind() == ErrorKind::NotFound => None,
+ Err(e) => return Err(e.into()),
+ };
+
+ let network_file = match fs::read(dump.path().join("network.json")) {
+ Ok(network_file) => Some(network_file),
+ Err(error) => match error.kind() {
+ // Allows the file to be missing, this will only result in all experimental features disabled.
+ ErrorKind::NotFound => {
+ debug!("`network.json` not found in dump");
+ None
+ }
+ _ => return Err(error.into()),
+ },
+ };
+ let network = if let Some(network_file) = network_file {
+ Some(serde_json::from_reader(&*network_file)?)
+ } else {
+ None
+ };
Ok(V6Reader {
metadata: serde_json::from_reader(&*meta_file)?,
instance_uid,
tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
+ batches,
keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
features,
+ network,
dump,
})
}
@@ -124,7 +156,7 @@ impl V6Reader {
&mut self,
) -> Box>)>> + '_> {
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
- let task: Task = serde_json::from_str(&line?).unwrap();
+ let task: Task = serde_json::from_str(&line?)?;
let update_file_path = self
.dump
@@ -136,8 +168,7 @@ impl V6Reader {
if update_file_path.exists() {
Ok((
task,
- Some(Box::new(UpdateFile::new(&update_file_path).unwrap())
- as Box),
+ Some(Box::new(UpdateFile::new(&update_file_path)?) as Box),
))
} else {
Ok((task, None))
@@ -145,6 +176,16 @@ impl V6Reader {
}))
}
+ pub fn batches(&mut self) -> Box> + '_> {
+ match self.batches.as_mut() {
+ Some(batches) => Box::new((batches).lines().map(|line| -> Result<_> {
+ let batch = serde_json::from_str(&line?)?;
+ Ok(batch)
+ })),
+ None => Box::new(std::iter::empty()) as Box> + '_>,
+ }
+ }
+
pub fn keys(&mut self) -> Box> + '_> {
Box::new(
(&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
@@ -154,6 +195,10 @@ impl V6Reader {
pub fn features(&self) -> Option {
self.features
}
+
+ pub fn network(&self) -> Option<&Network> {
+ self.network.as_ref()
+ }
}
pub struct UpdateFile {
@@ -210,7 +255,29 @@ impl V6IndexReader {
}
pub fn settings(&mut self) -> Result> {
- let settings: Settings = serde_json::from_reader(&mut self.settings)?;
+ let mut settings: Settings = serde_json::from_reader(&mut self.settings)?;
+ patch_embedders(&mut settings);
Ok(settings.check())
}
}
+
+fn patch_embedders(settings: &mut Settings) {
+ if let Setting::Set(embedders) = &mut settings.embedders {
+ for settings in embedders.values_mut() {
+ let Setting::Set(settings) = &mut settings.inner else {
+ continue;
+ };
+ if settings.source != Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace)
+ {
+ continue;
+ }
+ settings.pooling = match settings.pooling {
+ Setting::Set(pooling) => Setting::Set(pooling),
+ // if the pooling for a hugging face embedder is not set, force it to `forceMean`
+ // for backward compatibility with v1.13
+ // dumps created in v1.14 and up will have the setting set for hugging face embedders
+ Setting::Reset | Setting::NotSet => Setting::Set(OverridePooling::ForceMean),
+ };
+ }
+ }
+}
diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs
index 3ee51cabf..63b006b5c 100644
--- a/crates/dump/src/writer.rs
+++ b/crates/dump/src/writer.rs
@@ -4,7 +4,8 @@ use std::path::PathBuf;
use flate2::write::GzEncoder;
use flate2::Compression;
-use meilisearch_types::features::RuntimeTogglableFeatures;
+use meilisearch_types::batches::Batch;
+use meilisearch_types::features::{Network, RuntimeTogglableFeatures};
use meilisearch_types::keys::Key;
use meilisearch_types::settings::{Checked, Settings};
use serde_json::{Map, Value};
@@ -54,6 +55,10 @@ impl DumpWriter {
TaskWriter::new(self.dir.path().join("tasks"))
}
+ pub fn create_batches_queue(&self) -> Result {
+ BatchWriter::new(self.dir.path().join("batches"))
+ }
+
pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
Ok(std::fs::write(
self.dir.path().join("experimental-features.json"),
@@ -61,6 +66,10 @@ impl DumpWriter {
)?)
}
+ pub fn create_network(&self, network: Network) -> Result<()> {
+ Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?)
+ }
+
pub fn persist_to(self, mut writer: impl Write) -> Result<()> {
let gz_encoder = GzEncoder::new(&mut writer, Compression::default());
let mut tar_encoder = tar::Builder::new(gz_encoder);
@@ -84,7 +93,7 @@ impl KeyWriter {
}
pub fn push_key(&mut self, key: &Key) -> Result<()> {
- self.keys.write_all(&serde_json::to_vec(key)?)?;
+ serde_json::to_writer(&mut self.keys, &key)?;
self.keys.write_all(b"\n")?;
Ok(())
}
@@ -114,7 +123,7 @@ impl TaskWriter {
/// Pushes tasks in the dump.
/// If the tasks has an associated `update_file` it'll use the `task_id` as its name.
pub fn push_task(&mut self, task: &TaskDump) -> Result {
- self.queue.write_all(&serde_json::to_vec(task)?)?;
+ serde_json::to_writer(&mut self.queue, &task)?;
self.queue.write_all(b"\n")?;
Ok(UpdateFile::new(self.update_files.join(format!("{}.jsonl", task.uid))))
@@ -126,6 +135,30 @@ impl TaskWriter {
}
}
+pub struct BatchWriter {
+ queue: BufWriter,
+}
+
+impl BatchWriter {
+ pub(crate) fn new(path: PathBuf) -> Result {
+ std::fs::create_dir(&path)?;
+ let queue = File::create(path.join("queue.jsonl"))?;
+ Ok(BatchWriter { queue: BufWriter::new(queue) })
+ }
+
+ /// Pushes batches in the dump.
+ pub fn push_batch(&mut self, batch: &Batch) -> Result<()> {
+ serde_json::to_writer(&mut self.queue, &batch)?;
+ self.queue.write_all(b"\n")?;
+ Ok(())
+ }
+
+ pub fn flush(mut self) -> Result<()> {
+ self.queue.flush()?;
+ Ok(())
+ }
+}
+
pub struct UpdateFile {
path: PathBuf,
writer: Option>,
@@ -137,8 +170,8 @@ impl UpdateFile {
}
pub fn push_document(&mut self, document: &Document) -> Result<()> {
- if let Some(writer) = self.writer.as_mut() {
- writer.write_all(&serde_json::to_vec(document)?)?;
+ if let Some(mut writer) = self.writer.as_mut() {
+ serde_json::to_writer(&mut writer, &document)?;
writer.write_all(b"\n")?;
} else {
let file = File::create(&self.path).unwrap();
@@ -205,8 +238,8 @@ pub(crate) mod test {
use super::*;
use crate::reader::Document;
use crate::test::{
- create_test_api_keys, create_test_documents, create_test_dump, create_test_instance_uid,
- create_test_settings, create_test_tasks,
+ create_test_api_keys, create_test_batches, create_test_documents, create_test_dump,
+ create_test_instance_uid, create_test_settings, create_test_tasks,
};
fn create_directory_hierarchy(dir: &Path) -> String {
@@ -281,8 +314,10 @@ pub(crate) mod test {
let dump_path = dump.path();
// ==== checking global file hierarchy (we want to be sure there isn't too many files or too few)
- insta::assert_snapshot!(create_directory_hierarchy(dump_path), @r###"
+ insta::assert_snapshot!(create_directory_hierarchy(dump_path), @r"
.
+ ā---- batches/
+ ā ā---- queue.jsonl
ā---- indexes/
ā ā---- doggos/
ā ā ā---- documents.jsonl
@@ -295,8 +330,9 @@ pub(crate) mod test {
ā---- experimental-features.json
ā---- instance_uid.uuid
ā---- keys.jsonl
- ā---- metadata.json
- "###);
+ ā---- metadata.json
+ ā---- network.json
+ ");
// ==== checking the top level infos
let metadata = fs::read_to_string(dump_path.join("metadata.json")).unwrap();
@@ -349,6 +385,16 @@ pub(crate) mod test {
}
}
+ // ==== checking the batch queue
+ let batches_queue = fs::read_to_string(dump_path.join("batches/queue.jsonl")).unwrap();
+ for (batch, expected) in batches_queue.lines().zip(create_test_batches()) {
+ let mut batch = serde_json::from_str::(batch).unwrap();
+ if batch.details.settings == Some(Box::new(Settings::::default())) {
+ batch.details.settings = None;
+ }
+ assert_eq!(batch, expected, "{batch:#?}{expected:#?}");
+ }
+
// ==== checking the keys
let keys = fs::read_to_string(dump_path.join("keys.jsonl")).unwrap();
for (key, expected) in keys.lines().zip(create_test_api_keys()) {
diff --git a/crates/dump/tests/assets/v6-with-network.dump b/crates/dump/tests/assets/v6-with-network.dump
new file mode 100644
index 000000000..4d0d9ddc9
Binary files /dev/null and b/crates/dump/tests/assets/v6-with-network.dump differ
diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.toml
index 08b7bb717..66ea65336 100644
--- a/crates/file-store/Cargo.toml
+++ b/crates/file-store/Cargo.toml
@@ -11,7 +11,7 @@ edition.workspace = true
license.workspace = true
[dependencies]
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-tracing = "0.1.40"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
+tempfile = "3.15.0"
+thiserror = "2.0.9"
+tracing = "0.1.41"
+uuid = { version = "1.11.0", features = ["serde", "v4"] }
diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rs
index c8b3849ab..39ed9482b 100644
--- a/crates/file-store/src/lib.rs
+++ b/crates/file-store/src/lib.rs
@@ -136,6 +136,14 @@ pub struct File {
}
impl File {
+ pub fn from_parts(path: PathBuf, file: Option) -> Self {
+ Self { path, file }
+ }
+
+ pub fn into_parts(self) -> (PathBuf, Option) {
+ (self.path, self.file)
+ }
+
pub fn dry_file() -> Result {
Ok(Self { path: PathBuf::new(), file: None })
}
diff --git a/crates/filter-parser/Cargo.toml b/crates/filter-parser/Cargo.toml
index 6725a35d1..2657315a4 100644
--- a/crates/filter-parser/Cargo.toml
+++ b/crates/filter-parser/Cargo.toml
@@ -17,4 +17,5 @@ nom_locate = "4.2.0"
unescaper = "0.1.5"
[dev-dependencies]
-insta = "1.39.0"
+# fixed version due to format breakages in v1.40
+insta = "=1.39.0"
diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs
index 04b6dc266..0fc007bf1 100644
--- a/crates/filter-parser/src/condition.rs
+++ b/crates/filter-parser/src/condition.rs
@@ -30,6 +30,25 @@ pub enum Condition<'a> {
StartsWith { keyword: Token<'a>, word: Token<'a> },
}
+impl Condition<'_> {
+ pub fn operator(&self) -> &str {
+ match self {
+ Condition::GreaterThan(_) => ">",
+ Condition::GreaterThanOrEqual(_) => ">=",
+ Condition::Equal(_) => "=",
+ Condition::NotEqual(_) => "!=",
+ Condition::Null => "IS NULL",
+ Condition::Empty => "IS EMPTY",
+ Condition::Exists => "EXISTS",
+ Condition::LowerThan(_) => "<",
+ Condition::LowerThanOrEqual(_) => "<=",
+ Condition::Between { .. } => "TO",
+ Condition::Contains { .. } => "CONTAINS",
+ Condition::StartsWith { .. } => "STARTS WITH",
+ }
+ }
+}
+
/// condition = value ("==" | ">" ...) value
pub fn parse_condition(input: Span) -> IResult {
let operator = alt((tag("<="), tag(">="), tag("!="), tag("<"), tag(">"), tag("=")));
diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs
index 122396b87..855ce983e 100644
--- a/crates/filter-parser/src/error.rs
+++ b/crates/filter-parser/src/error.rs
@@ -35,7 +35,7 @@ impl NomErrorExt for nom::Err {
pub fn cut_with_err<'a, O>(
mut parser: impl FnMut(Span<'a>) -> IResult<'a, O>,
mut with: impl FnMut(Error<'a>) -> Error<'a>,
-) -> impl FnMut(Span<'a>) -> IResult {
+) -> impl FnMut(Span<'a>) -> IResult<'a, O> {
move |input| match parser.parse(input) {
Err(nom::Err::Error(e)) => Err(nom::Err::Failure(with(e))),
rest => rest,
@@ -121,7 +121,7 @@ impl<'a> ParseError> for Error<'a> {
}
}
-impl<'a> Display for Error<'a> {
+impl Display for Error<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let input = self.context.fragment();
// When printing our error message we want to escape all `\n` to be sure we keep our format with the
diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs
index cfe009acb..938702103 100644
--- a/crates/filter-parser/src/lib.rs
+++ b/crates/filter-parser/src/lib.rs
@@ -80,7 +80,7 @@ pub struct Token<'a> {
value: Option,
}
-impl<'a> PartialEq for Token<'a> {
+impl PartialEq for Token<'_> {
fn eq(&self, other: &Self) -> bool {
self.span.fragment() == other.span.fragment()
}
@@ -179,6 +179,26 @@ impl<'a> FilterCondition<'a> {
}
}
+ pub fn fids(&self, depth: usize) -> Box + '_> {
+ if depth == 0 {
+ return Box::new(std::iter::empty());
+ }
+ match self {
+ FilterCondition::Condition { fid, .. } | FilterCondition::In { fid, .. } => {
+ Box::new(std::iter::once(fid))
+ }
+ FilterCondition::Not(filter) => {
+ let depth = depth.saturating_sub(1);
+ filter.fids(depth)
+ }
+ FilterCondition::And(subfilters) | FilterCondition::Or(subfilters) => {
+ let depth = depth.saturating_sub(1);
+ Box::new(subfilters.iter().flat_map(move |f| f.fids(depth)))
+ }
+ _ => Box::new(std::iter::empty()),
+ }
+ }
+
/// Returns the first token found at the specified depth, `None` if no token at this depth.
pub fn token_at_depth(&self, depth: usize) -> Option<&Token> {
match self {
@@ -206,7 +226,7 @@ impl<'a> FilterCondition<'a> {
}
}
- pub fn parse(input: &'a str) -> Result, Error> {
+ pub fn parse(input: &'a str) -> Result, Error<'a>> {
if input.trim().is_empty() {
return Ok(None);
}
@@ -507,7 +527,7 @@ pub fn parse_filter(input: Span) -> IResult {
terminated(|input| parse_expression(input, 0), eof)(input)
}
-impl<'a> std::fmt::Display for FilterCondition<'a> {
+impl std::fmt::Display for FilterCondition<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FilterCondition::Not(filter) => {
@@ -556,7 +576,8 @@ impl<'a> std::fmt::Display for FilterCondition<'a> {
}
}
}
-impl<'a> std::fmt::Display for Condition<'a> {
+
+impl std::fmt::Display for Condition<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Condition::GreaterThan(token) => write!(f, "> {token}"),
@@ -574,7 +595,8 @@ impl<'a> std::fmt::Display for Condition<'a> {
}
}
}
-impl<'a> std::fmt::Display for Token<'a> {
+
+impl std::fmt::Display for Token<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{{{}}}", self.value())
}
@@ -978,6 +1000,43 @@ pub mod tests {
assert!(filter.token_at_depth(3).is_none());
}
+ #[test]
+ fn fids() {
+ let filter = Fc::parse("field = value").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(MAX_FILTER_DEPTH).collect();
+ assert_eq!(fids.len(), 1);
+ assert_eq!(fids[0].value(), "field");
+
+ let filter = Fc::parse("field IN [1, 2, 3]").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(MAX_FILTER_DEPTH).collect();
+ assert_eq!(fids.len(), 1);
+ assert_eq!(fids[0].value(), "field");
+
+ let filter = Fc::parse("field != value").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(MAX_FILTER_DEPTH).collect();
+ assert_eq!(fids.len(), 1);
+ assert_eq!(fids[0].value(), "field");
+
+ let filter = Fc::parse("field1 = value1 AND field2 = value2").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(MAX_FILTER_DEPTH).collect();
+ assert_eq!(fids.len(), 2);
+ assert!(fids[0].value() == "field1");
+ assert!(fids[1].value() == "field2");
+
+ let filter = Fc::parse("field1 = value1 OR field2 = value2").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(MAX_FILTER_DEPTH).collect();
+ assert_eq!(fids.len(), 2);
+ assert!(fids[0].value() == "field1");
+ assert!(fids[1].value() == "field2");
+
+ let depth = 2;
+ let filter =
+ Fc::parse("field1 = value1 AND (field2 = value2 OR field3 = value3)").unwrap().unwrap();
+ let fids: Vec<_> = filter.fids(depth).collect();
+ assert_eq!(fids.len(), 1);
+ assert_eq!(fids[0].value(), "field1");
+ }
+
#[test]
fn token_from_str() {
let s = "test string that should not be parsed";
diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs
index 5912f6900..98cac39fe 100644
--- a/crates/filter-parser/src/value.rs
+++ b/crates/filter-parser/src/value.rs
@@ -52,7 +52,7 @@ fn quoted_by(quote: char, input: Span) -> IResult {
}
// word = (alphanumeric | _ | - | .)+ except for reserved keywords
-pub fn word_not_keyword<'a>(input: Span<'a>) -> IResult> {
+pub fn word_not_keyword<'a>(input: Span<'a>) -> IResult<'a, Token<'a>> {
let (input, word): (_, Token<'a>) =
take_while1(is_value_component)(input).map(|(s, t)| (s, t.into()))?;
if is_keyword(word.value()) {
diff --git a/crates/fuzzers/Cargo.toml b/crates/fuzzers/Cargo.toml
index 86a0f779d..a838350ba 100644
--- a/crates/fuzzers/Cargo.toml
+++ b/crates/fuzzers/Cargo.toml
@@ -11,12 +11,12 @@ edition.workspace = true
license.workspace = true
[dependencies]
-arbitrary = { version = "1.3.2", features = ["derive"] }
+arbitrary = { version = "1.4.1", features = ["derive"] }
bumpalo = "3.16.0"
-clap = { version = "4.5.9", features = ["derive"] }
+clap = { version = "4.5.24", features = ["derive"] }
either = "1.13.0"
-fastrand = "2.1.0"
+fastrand = "2.3.0"
milli = { path = "../milli" }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
-tempfile = "3.10.1"
+serde = { version = "1.0.217", features = ["derive"] }
+serde_json = { version = "1.0.135", features = ["preserve_order"] }
+tempfile = "3.15.0"
diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs
index f335938b9..4df989b51 100644
--- a/crates/fuzzers/src/bin/fuzz-indexing.rs
+++ b/crates/fuzzers/src/bin/fuzz-indexing.rs
@@ -10,8 +10,9 @@ use either::Either;
use fuzzers::Operation;
use milli::documents::mmap_from_objects;
use milli::heed::EnvOpenOptions;
+use milli::progress::Progress;
use milli::update::new::indexer;
-use milli::update::{IndexDocumentsMethod, IndexerConfig};
+use milli::update::IndexerConfig;
use milli::vector::EmbeddingConfigs;
use milli::Index;
use serde_json::Value;
@@ -56,13 +57,14 @@ fn main() {
let opt = opt.clone();
let handle = std::thread::spawn(move || {
- let mut options = EnvOpenOptions::new();
+ let options = EnvOpenOptions::new();
+ let mut options = options.read_txn_without_tls();
options.map_size(1024 * 1024 * 1024 * 1024);
let tempdir = match opt.path {
Some(path) => TempDir::new_in(path).unwrap(),
None => TempDir::new().unwrap(),
};
- let index = Index::new(options, tempdir.path()).unwrap();
+ let index = Index::new(options, tempdir.path(), true).unwrap();
let indexer_config = IndexerConfig::default();
std::thread::scope(|s| {
@@ -88,9 +90,7 @@ fn main() {
let indexer_alloc = Bump::new();
let embedders = EmbeddingConfigs::default();
- let mut indexer = indexer::DocumentOperation::new(
- IndexDocumentsMethod::ReplaceDocuments,
- );
+ let mut indexer = indexer::DocumentOperation::new();
let mut operations = Vec::new();
for op in batch.0 {
@@ -114,7 +114,7 @@ fn main() {
for op in &operations {
match op {
Either::Left(documents) => {
- indexer.add_documents(documents).unwrap()
+ indexer.replace_documents(documents).unwrap()
}
Either::Right(ids) => indexer.delete_documents(ids),
}
@@ -128,13 +128,14 @@ fn main() {
None,
&mut new_fields_ids_map,
&|| false,
- &|_progress| (),
+ Progress::default(),
)
.unwrap();
indexer::index(
&mut wtxn,
&index,
+ &milli::ThreadPoolNoAbortBuilder::new().build().unwrap(),
indexer_config.grenad_parameters(),
&db_fields_ids_map,
new_fields_ids_map,
@@ -142,7 +143,7 @@ fn main() {
&document_changes,
embedders,
&|| false,
- &|_| (),
+ &Progress::default(),
)
.unwrap();
diff --git a/crates/index-scheduler/Cargo.toml b/crates/index-scheduler/Cargo.toml
index 657dd6dfe..37b3ea835 100644
--- a/crates/index-scheduler/Cargo.toml
+++ b/crates/index-scheduler/Cargo.toml
@@ -11,41 +11,42 @@ edition.workspace = true
license.workspace = true
[dependencies]
-anyhow = "1.0.86"
+anyhow = "1.0.95"
bincode = "1.3.3"
-csv = "1.3.0"
-derive_builder = "0.20.0"
+bumpalo = "3.16.0"
+bumparaw-collections = "0.1.4"
+convert_case = "0.6.0"
+csv = "1.3.1"
+derive_builder = "0.20.2"
dump = { path = "../dump" }
enum-iterator = "2.1.0"
file-store = { path = "../file-store" }
-flate2 = "1.0.30"
+flate2 = "1.0.35"
meilisearch-auth = { path = "../meilisearch-auth" }
meilisearch-types = { path = "../meilisearch-types" }
+memmap2 = "0.9.5"
page_size = "0.6.0"
-raw-collections = { git = "https://github.com/meilisearch/raw-collections.git", version = "0.1.0" }
rayon = "1.10.0"
-roaring = { version = "0.10.6", features = ["serde"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
+roaring = { version = "0.10.10", features = ["serde"] }
+serde = { version = "1.0.217", features = ["derive"] }
+serde_json = { version = "1.0.138", features = ["preserve_order"] }
synchronoise = "1.0.1"
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-memmap2 = "0.9.4"
-time = { version = "0.3.36", features = [
+tempfile = "3.15.0"
+thiserror = "2.0.9"
+time = { version = "0.3.37", features = [
"serde-well-known",
"formatting",
"parsing",
"macros",
] }
-tracing = "0.1.40"
-ureq = "2.10.0"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
-bumpalo = "3.16.0"
+tracing = "0.1.41"
+ureq = "2.12.1"
+uuid = { version = "1.11.0", features = ["serde", "v4"] }
[dev-dependencies]
-arroy = "0.5.0"
big_s = "1.0.2"
-crossbeam = "0.8.4"
-insta = { version = "1.39.0", features = ["json", "redactions"] }
+crossbeam-channel = "0.5.14"
+# fixed version due to format breakages in v1.40
+insta = { version = "=1.39.0", features = ["json", "redactions"] }
maplit = "1.0.2"
meili-snap = { path = "../meili-snap" }
diff --git a/crates/index-scheduler/src/autobatcher.rs b/crates/index-scheduler/src/autobatcher.rs
deleted file mode 100644
index 0f6aa8a3a..000000000
--- a/crates/index-scheduler/src/autobatcher.rs
+++ /dev/null
@@ -1,1019 +0,0 @@
-/*!
-The autobatcher is responsible for combining the next enqueued
-tasks affecting a single index into a [batch](crate::batch::Batch).
-
-The main function of the autobatcher is [`next_autobatch`].
-*/
-
-use std::ops::ControlFlow::{self, Break, Continue};
-
-use meilisearch_types::milli::update::IndexDocumentsMethod::{
- self, ReplaceDocuments, UpdateDocuments,
-};
-use meilisearch_types::tasks::TaskId;
-
-use crate::KindWithContent;
-
-/// Succinctly describes a task's [`Kind`](meilisearch_types::tasks::Kind)
-/// for the purpose of simplifying the implementation of the autobatcher.
-///
-/// Only the non-prioritised tasks that can be grouped in a batch have a corresponding [`AutobatchKind`]
-enum AutobatchKind {
- DocumentImport {
- method: IndexDocumentsMethod,
- allow_index_creation: bool,
- primary_key: Option,
- },
- DocumentEdition,
- DocumentDeletion {
- by_filter: bool,
- },
- DocumentClear,
- Settings {
- allow_index_creation: bool,
- },
- IndexCreation,
- IndexDeletion,
- IndexUpdate,
- IndexSwap,
-}
-
-impl AutobatchKind {
- #[rustfmt::skip]
- fn allow_index_creation(&self) -> Option {
- match self {
- AutobatchKind::DocumentImport { allow_index_creation, .. }
- | AutobatchKind::Settings { allow_index_creation, .. } => Some(*allow_index_creation),
- _ => None,
- }
- }
-
- fn primary_key(&self) -> Option> {
- match self {
- AutobatchKind::DocumentImport { primary_key, .. } => Some(primary_key.as_deref()),
- _ => None,
- }
- }
-}
-
-impl From for AutobatchKind {
- fn from(kind: KindWithContent) -> Self {
- match kind {
- KindWithContent::DocumentAdditionOrUpdate {
- method,
- allow_index_creation,
- primary_key,
- ..
- } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
- KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
- KindWithContent::DocumentDeletion { .. } => {
- AutobatchKind::DocumentDeletion { by_filter: false }
- }
- KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
- KindWithContent::DocumentDeletionByFilter { .. } => {
- AutobatchKind::DocumentDeletion { by_filter: true }
- }
- KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
- AutobatchKind::Settings {
- allow_index_creation: allow_index_creation && !is_deletion,
- }
- }
- KindWithContent::IndexDeletion { .. } => AutobatchKind::IndexDeletion,
- KindWithContent::IndexCreation { .. } => AutobatchKind::IndexCreation,
- KindWithContent::IndexUpdate { .. } => AutobatchKind::IndexUpdate,
- KindWithContent::IndexSwap { .. } => AutobatchKind::IndexSwap,
- KindWithContent::TaskCancelation { .. }
- | KindWithContent::TaskDeletion { .. }
- | KindWithContent::DumpCreation { .. }
- | KindWithContent::SnapshotCreation => {
- panic!("The autobatcher should never be called with tasks that don't apply to an index.")
- }
- }
- }
-}
-
-#[derive(Debug)]
-pub enum BatchKind {
- DocumentClear {
- ids: Vec,
- },
- DocumentOperation {
- method: IndexDocumentsMethod,
- allow_index_creation: bool,
- primary_key: Option,
- operation_ids: Vec,
- },
- DocumentEdition {
- id: TaskId,
- },
- DocumentDeletion {
- deletion_ids: Vec,
- includes_by_filter: bool,
- },
- ClearAndSettings {
- other: Vec,
- allow_index_creation: bool,
- settings_ids: Vec,
- },
- SettingsAndDocumentOperation {
- settings_ids: Vec,
- method: IndexDocumentsMethod,
- allow_index_creation: bool,
- primary_key: Option,
- operation_ids: Vec,
- },
- Settings {
- allow_index_creation: bool,
- settings_ids: Vec,
- },
- IndexDeletion {
- ids: Vec,
- },
- IndexCreation {
- id: TaskId,
- },
- IndexUpdate {
- id: TaskId,
- },
- IndexSwap {
- id: TaskId,
- },
-}
-
-impl BatchKind {
- #[rustfmt::skip]
- fn allow_index_creation(&self) -> Option {
- match self {
- BatchKind::DocumentOperation { allow_index_creation, .. }
- | BatchKind::ClearAndSettings { allow_index_creation, .. }
- | BatchKind::SettingsAndDocumentOperation { allow_index_creation, .. }
- | BatchKind::Settings { allow_index_creation, .. } => Some(*allow_index_creation),
- _ => None,
- }
- }
-
- fn primary_key(&self) -> Option> {
- match self {
- BatchKind::DocumentOperation { primary_key, .. }
- | BatchKind::SettingsAndDocumentOperation { primary_key, .. } => {
- Some(primary_key.as_deref())
- }
- _ => None,
- }
- }
-}
-
-impl BatchKind {
- /// Returns a `ControlFlow::Break` if you must stop right now.
- /// The boolean tell you if an index has been created by the batched task.
- /// To ease the writing of the code. `true` can be returned when you don't need to create an index
- /// but false can't be returned if you needs to create an index.
- // TODO use an AutoBatchKind as input
- pub fn new(
- task_id: TaskId,
- kind: KindWithContent,
- primary_key: Option<&str>,
- ) -> (ControlFlow, bool) {
- use AutobatchKind as K;
-
- match AutobatchKind::from(kind) {
- K::IndexCreation => (Break(BatchKind::IndexCreation { id: task_id }), true),
- K::IndexDeletion => (Break(BatchKind::IndexDeletion { ids: vec![task_id] }), false),
- K::IndexUpdate => (Break(BatchKind::IndexUpdate { id: task_id }), false),
- K::IndexSwap => (Break(BatchKind::IndexSwap { id: task_id }), false),
- K::DocumentClear => (Continue(BatchKind::DocumentClear { ids: vec![task_id] }), false),
- K::DocumentImport { method, allow_index_creation, primary_key: pk }
- if primary_key.is_none() || pk.is_none() || primary_key == pk.as_deref() =>
- {
- (
- Continue(BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key: pk,
- operation_ids: vec![task_id],
- }),
- allow_index_creation,
- )
- }
- // if the primary key set in the task was different than ours we should stop and make this batch fail asap.
- K::DocumentImport { method, allow_index_creation, primary_key } => (
- Break(BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key,
- operation_ids: vec![task_id],
- }),
- allow_index_creation,
- ),
- K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
- K::DocumentDeletion { by_filter: includes_by_filter } => (
- Continue(BatchKind::DocumentDeletion {
- deletion_ids: vec![task_id],
- includes_by_filter,
- }),
- false,
- ),
- K::Settings { allow_index_creation } => (
- Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
- allow_index_creation,
- ),
- }
- }
-
- /// Returns a `ControlFlow::Break` if you must stop right now.
- /// The boolean tell you if an index has been created by the batched task.
- /// To ease the writing of the code. `true` can be returned when you don't need to create an index
- /// but false can't be returned if you needs to create an index.
- #[rustfmt::skip]
- fn accumulate(self, id: TaskId, kind: AutobatchKind, index_already_exists: bool, primary_key: Option<&str>) -> ControlFlow {
- use AutobatchKind as K;
-
- match (self, kind) {
- // We don't batch any of these operations
- (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break(this),
- // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
- (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
- Break(this)
- },
- // NOTE: We need to negate the whole condition since we're checking if we need to break instead of continue.
- // I wrote it this way because it's easier to understand than the other way around.
- (this, kind) if !(
- // 1. If both task don't interact with primary key -> we can continue
- (this.primary_key().is_none() && kind.primary_key().is_none()) ||
- // 2. Else ->
- (
- // 2.1 If we already have a primary-key ->
- (
- primary_key.is_some() &&
- // 2.1.1 If the task we're trying to accumulate have a pk it must be equal to our primary key
- // 2.1.2 If the task don't have a primary-key -> we can continue
- kind.primary_key().map_or(true, |pk| pk == primary_key)
- ) ||
- // 2.2 If we don't have a primary-key ->
- (
- // 2.2.1 If both the batch and the task have a primary key they should be equal
- // 2.2.2 If the batch is set to Some(None), the task should be too
- // 2.2.3 If the batch is set to None -> we can continue
- this.primary_key().zip(kind.primary_key()).map_or(true, |(this, kind)| this == kind)
- )
- )
-
- ) // closing the negation
-
- => {
- Break(this)
- },
- // The index deletion can batch with everything but must stop after
- (
- BatchKind::DocumentClear { mut ids }
- | BatchKind::DocumentDeletion { deletion_ids: mut ids, includes_by_filter: _ }
- | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
- | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
- K::IndexDeletion,
- ) => {
- ids.push(id);
- Break(BatchKind::IndexDeletion { ids })
- }
- (
- BatchKind::ClearAndSettings { settings_ids: mut ids, allow_index_creation: _, mut other }
- | BatchKind::SettingsAndDocumentOperation { operation_ids: mut ids, method: _, allow_index_creation: _, primary_key: _, settings_ids: mut other },
- K::IndexDeletion,
- ) => {
- ids.push(id);
- ids.append(&mut other);
- Break(BatchKind::IndexDeletion { ids })
- }
-
- (
- BatchKind::DocumentClear { mut ids },
- K::DocumentClear | K::DocumentDeletion { by_filter: _ },
- ) => {
- ids.push(id);
- Continue(BatchKind::DocumentClear { ids })
- }
- (
- this @ BatchKind::DocumentClear { .. },
- K::DocumentImport { .. } | K::Settings { .. },
- ) => Break(this),
- (
- BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, mut operation_ids },
- K::DocumentClear,
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::DocumentClear { ids: operation_ids })
- }
-
- // we can autobatch the same kind of document additions / updates
- (
- BatchKind::DocumentOperation { method: ReplaceDocuments, allow_index_creation, primary_key: _, mut operation_ids },
- K::DocumentImport { method: ReplaceDocuments, primary_key: pk, .. },
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::DocumentOperation {
- method: ReplaceDocuments,
- allow_index_creation,
- operation_ids,
- primary_key: pk,
- })
- }
- (
- BatchKind::DocumentOperation { method: UpdateDocuments, allow_index_creation, primary_key: _, mut operation_ids },
- K::DocumentImport { method: UpdateDocuments, primary_key: pk, .. },
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::DocumentOperation {
- method: UpdateDocuments,
- allow_index_creation,
- primary_key: pk,
- operation_ids,
- })
- }
- (
- BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
- K::DocumentDeletion { by_filter: false },
- ) => {
- operation_ids.push(id);
-
- Continue(BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key,
- operation_ids,
- })
- }
- // We can't batch a document operation with a delete by filter
- (
- this @ BatchKind::DocumentOperation { .. },
- K::DocumentDeletion { by_filter: true },
- ) => {
- Break(this)
- }
- // but we can't autobatch documents if it's not the same kind
- // this match branch MUST be AFTER the previous one
- (
- this @ BatchKind::DocumentOperation { .. },
- K::DocumentImport { .. },
- ) => Break(this),
-
- (
- BatchKind::DocumentOperation { method, allow_index_creation, primary_key, operation_ids },
- K::Settings { .. },
- ) => Continue(BatchKind::SettingsAndDocumentOperation {
- settings_ids: vec![id],
- method,
- allow_index_creation,
- primary_key,
- operation_ids,
- }),
-
- (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: _ }, K::DocumentClear) => {
- deletion_ids.push(id);
- Continue(BatchKind::DocumentClear { ids: deletion_ids })
- }
- // we can't autobatch the deletion and import if the document deletion contained a filter
- (
- this @ BatchKind::DocumentDeletion { deletion_ids: _, includes_by_filter: true },
- K::DocumentImport { .. }
- ) => Break(this),
- // we can autobatch the deletion and import if the index already exists
- (
- BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
- K::DocumentImport { method, allow_index_creation, primary_key }
- ) if index_already_exists => {
- deletion_ids.push(id);
-
- Continue(BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key,
- operation_ids: deletion_ids,
- })
- }
- // we can autobatch the deletion and import if both can't create an index
- (
- BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
- K::DocumentImport { method, allow_index_creation, primary_key }
- ) if !allow_index_creation => {
- deletion_ids.push(id);
-
- Continue(BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key,
- operation_ids: deletion_ids,
- })
- }
- // we can't autobatch a deletion and an import if the index does not exists but would be created by an addition
- (
- this @ BatchKind::DocumentDeletion { .. },
- K::DocumentImport { .. }
- ) => {
- Break(this)
- }
- (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter }, K::DocumentDeletion { by_filter }) => {
- deletion_ids.push(id);
- Continue(BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: includes_by_filter | by_filter })
- }
- (this @ BatchKind::DocumentDeletion { .. }, K::Settings { .. }) => Break(this),
-
- (
- BatchKind::Settings { settings_ids, allow_index_creation },
- K::DocumentClear,
- ) => Continue(BatchKind::ClearAndSettings {
- settings_ids,
- allow_index_creation,
- other: vec![id],
- }),
- (
- this @ BatchKind::Settings { .. },
- K::DocumentImport { .. } | K::DocumentDeletion { .. },
- ) => Break(this),
- (
- BatchKind::Settings { mut settings_ids, allow_index_creation },
- K::Settings { .. },
- ) => {
- settings_ids.push(id);
- Continue(BatchKind::Settings {
- allow_index_creation,
- settings_ids,
- })
- }
-
- (
- BatchKind::ClearAndSettings { mut other, settings_ids, allow_index_creation },
- K::DocumentClear,
- ) => {
- other.push(id);
- Continue(BatchKind::ClearAndSettings {
- other,
- settings_ids,
- allow_index_creation,
- })
- }
- (this @ BatchKind::ClearAndSettings { .. }, K::DocumentImport { .. }) => Break(this),
- (
- BatchKind::ClearAndSettings {
- mut other,
- settings_ids,
- allow_index_creation,
- },
- K::DocumentDeletion { .. },
- ) => {
- other.push(id);
- Continue(BatchKind::ClearAndSettings {
- other,
- settings_ids,
- allow_index_creation,
- })
- }
- (
- BatchKind::ClearAndSettings { mut settings_ids, other, allow_index_creation },
- K::Settings { .. },
- ) => {
- settings_ids.push(id);
- Continue(BatchKind::ClearAndSettings {
- other,
- settings_ids,
- allow_index_creation,
- })
- }
- (
- BatchKind::SettingsAndDocumentOperation { settings_ids, method: _, mut operation_ids, allow_index_creation, primary_key: _ },
- K::DocumentClear,
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::ClearAndSettings {
- settings_ids,
- other: operation_ids,
- allow_index_creation,
- })
- }
-
- (
- BatchKind::SettingsAndDocumentOperation { settings_ids, method: ReplaceDocuments, mut operation_ids, allow_index_creation, primary_key: _},
- K::DocumentImport { method: ReplaceDocuments, primary_key: pk2, .. },
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::SettingsAndDocumentOperation {
- settings_ids,
- method: ReplaceDocuments,
- allow_index_creation,
- primary_key: pk2,
- operation_ids,
- })
- }
- (
- BatchKind::SettingsAndDocumentOperation { settings_ids, method: UpdateDocuments, allow_index_creation, primary_key: _, mut operation_ids },
- K::DocumentImport { method: UpdateDocuments, primary_key: pk2, .. },
- ) => {
- operation_ids.push(id);
- Continue(BatchKind::SettingsAndDocumentOperation {
- settings_ids,
- method: UpdateDocuments,
- allow_index_creation,
- primary_key: pk2,
- operation_ids,
- })
- }
- // But we can't batch a settings and a doc op with another doc op
- // this MUST be AFTER the two previous branch
- (
- this @ BatchKind::SettingsAndDocumentOperation { .. },
- K::DocumentDeletion { .. } | K::DocumentImport { .. },
- ) => Break(this),
- (
- BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
- K::Settings { .. },
- ) => {
- settings_ids.push(id);
- Continue(BatchKind::SettingsAndDocumentOperation {
- settings_ids,
- method,
- allow_index_creation,
- primary_key,
- operation_ids,
- })
- }
- (
- BatchKind::IndexCreation { .. }
- | BatchKind::IndexDeletion { .. }
- | BatchKind::IndexUpdate { .. }
- | BatchKind::IndexSwap { .. }
- | BatchKind::DocumentEdition { .. },
- _,
- ) => {
- unreachable!()
- }
- }
- }
-}
-
-/// Create a batch from an ordered list of tasks.
-///
-/// ## Preconditions
-/// 1. The tasks must be enqueued and given in the order in which they were enqueued
-/// 2. The tasks must not be prioritised tasks (e.g. task cancellation, dump, snapshot, task deletion)
-/// 3. The tasks must all be related to the same index
-///
-/// ## Return
-/// `None` if the list of tasks is empty. Otherwise, an [`AutoBatch`] that represents
-/// a subset of the given tasks.
-pub fn autobatch(
- enqueued: Vec<(TaskId, KindWithContent)>,
- index_already_exists: bool,
- primary_key: Option<&str>,
-) -> Option<(BatchKind, bool)> {
- let mut enqueued = enqueued.into_iter();
- let (id, kind) = enqueued.next()?;
-
- // index_exist will keep track of if the index should exist at this point after the tasks we batched.
- let mut index_exist = index_already_exists;
-
- let (mut acc, must_create_index) = match BatchKind::new(id, kind, primary_key) {
- (Continue(acc), create) => (acc, create),
- (Break(acc), create) => return Some((acc, create)),
- };
-
- // if an index has been created in the previous step we can consider it as existing.
- index_exist |= must_create_index;
-
- for (id, kind) in enqueued {
- acc = match acc.accumulate(id, kind.into(), index_exist, primary_key) {
- Continue(acc) => acc,
- Break(acc) => return Some((acc, must_create_index)),
- };
- }
-
- Some((acc, must_create_index))
-}
-
-#[cfg(test)]
-mod tests {
- use meilisearch_types::tasks::IndexSwap;
- use uuid::Uuid;
-
- use super::*;
- use crate::debug_snapshot;
-
- fn autobatch_from(
- index_already_exists: bool,
- primary_key: Option<&str>,
- input: impl IntoIterator
- ,
- ) -> Option<(BatchKind, bool)> {
- autobatch(
- input.into_iter().enumerate().map(|(id, kind)| (id as TaskId, kind)).collect(),
- index_already_exists,
- primary_key,
- )
- }
-
- fn doc_imp(
- method: IndexDocumentsMethod,
- allow_index_creation: bool,
- primary_key: Option<&str>,
- ) -> KindWithContent {
- KindWithContent::DocumentAdditionOrUpdate {
- index_uid: String::from("doggo"),
- primary_key: primary_key.map(|pk| pk.to_string()),
- method,
- content_file: Uuid::new_v4(),
- documents_count: 0,
- allow_index_creation,
- }
- }
-
- fn doc_del() -> KindWithContent {
- KindWithContent::DocumentDeletion {
- index_uid: String::from("doggo"),
- documents_ids: Vec::new(),
- }
- }
-
- fn doc_del_fil() -> KindWithContent {
- KindWithContent::DocumentDeletionByFilter {
- index_uid: String::from("doggo"),
- filter_expr: serde_json::json!("cuteness > 100"),
- }
- }
-
- fn doc_clr() -> KindWithContent {
- KindWithContent::DocumentClear { index_uid: String::from("doggo") }
- }
-
- fn settings(allow_index_creation: bool) -> KindWithContent {
- KindWithContent::SettingsUpdate {
- index_uid: String::from("doggo"),
- new_settings: Default::default(),
- is_deletion: false,
- allow_index_creation,
- }
- }
-
- fn idx_create() -> KindWithContent {
- KindWithContent::IndexCreation { index_uid: String::from("doggo"), primary_key: None }
- }
-
- fn idx_update() -> KindWithContent {
- KindWithContent::IndexUpdate { index_uid: String::from("doggo"), primary_key: None }
- }
-
- fn idx_del() -> KindWithContent {
- KindWithContent::IndexDeletion { index_uid: String::from("doggo") }
- }
-
- fn idx_swap() -> KindWithContent {
- KindWithContent::IndexSwap {
- swaps: vec![IndexSwap { indexes: (String::from("doggo"), String::from("catto")) }],
- }
- }
-
- #[test]
- fn autobatch_simple_operation_together() {
- // we can autobatch one or multiple `ReplaceDocuments` together.
- // if the index exists.
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, false , None), doc_imp(ReplaceDocuments, false , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
-
- // if it doesn't exists.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-
- // we can autobatch one or multiple `UpdateDocuments` together.
- // if the index exists.
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
-
- // if it doesn't exists.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
-
- // we can autobatch one or multiple DocumentDeletion together
- debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
-
- // we can autobatch one or multiple DocumentDeletionByFilter together
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
-
- // we can autobatch one or multiple Settings together
- debug_snapshot!(autobatch_from(true, None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [settings(true), settings(true), settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [settings(false), settings(false), settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0, 1, 2] }, false))");
-
- debug_snapshot!(autobatch_from(false,None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false,None, [settings(true), settings(true), settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(false,None, [settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [settings(false), settings(false), settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0, 1, 2] }, false))");
-
- // We can autobatch document addition with document deletion
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- // And the other way around
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-
- // But we can't autobatch document addition with document deletion by filter
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
- debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
- // And the other way around
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- }
-
- #[test]
- fn simple_document_operation_dont_autobatch_with_other() {
- // addition, updates and deletion by filter can't batch together
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
- }
-
- #[test]
- fn document_addition_batch_with_settings() {
- // simple case
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-
- // multiple settings and doc addition
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-
- // addition and setting unordered
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 2] }, true))");
-
- // We ensure this kind of batch doesn't batch with forbidden operations
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- }
-
- #[test]
- fn clear_and_additions() {
- // these two doesn't need to batch
- debug_snapshot!(autobatch_from(true, None, [doc_clr(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentClear { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_clr(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentClear { ids: [0] }, false))");
-
- // Basic use case
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), doc_clr()]), @"Some((DocumentClear { ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_clr()]), @"Some((DocumentClear { ids: [0, 1, 2] }, true))");
-
- // This batch kind doesn't mix with other document addition
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), doc_clr(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentClear { ids: [0, 1, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_clr(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentClear { ids: [0, 1, 2] }, true))");
-
- // But you can batch multiple clear together
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), doc_clr(), doc_clr(), doc_clr()]), @"Some((DocumentClear { ids: [0, 1, 2, 3, 4] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_clr(), doc_clr(), doc_clr()]), @"Some((DocumentClear { ids: [0, 1, 2, 3, 4] }, true))");
- }
-
- #[test]
- fn clear_and_additions_and_settings() {
- // A clear don't need to autobatch the settings that happens AFTER there is no documents
- debug_snapshot!(autobatch_from(true, None, [doc_clr(), settings(true)]), @"Some((DocumentClear { ids: [0] }, false))");
-
- debug_snapshot!(autobatch_from(true, None, [settings(true), doc_clr(), settings(true)]), @"Some((ClearAndSettings { other: [1], allow_index_creation: true, settings_ids: [0, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_clr()]), @"Some((ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_clr()]), @"Some((ClearAndSettings { other: [0, 2], allow_index_creation: true, settings_ids: [1] }, true))");
- }
-
- #[test]
- fn anything_and_index_deletion() {
- // The `IndexDeletion` doesn't batch with anything that happens AFTER.
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
-
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
-
- // The index deletion can accept almost any type of `BatchKind` and transform it to an `IndexDeletion`.
- // First, the basic cases
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-
- // Then the mixed cases.
- // The index already exists, whatever is the right of the tasks it shouldn't change the result.
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,false, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,false, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,false, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,true, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments,true, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
-
- // When the index doesn't exists yet it's more complicated.
- // Either the first task we encounter create it, in which case we can create a big batch with everything.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- // The right of the tasks following isn't really important.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,true, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,true, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, true, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, true))");
- // Or, the second case; the first task doesn't create the index and thus we wants to batch it with only tasks that can't create an index.
- // that can be a second task that don't have the right to create an index. Or anything that can't create an index like an index deletion, document deletion, document clear, etc.
- // All theses tasks are going to throw an error `Index doesn't exist` once the batch is processed.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,false, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
- // The third and final case is when the first task doesn't create an index but is directly followed by a task creating an index. In this case we can't batch whit what
- // follows because we first need to process the erronous batch.
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments,false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(UpdateDocuments, false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- }
-
- #[test]
- fn allowed_and_disallowed_index_creation() {
- // `DocumentImport` can't be mixed with those disallowed to do so except if the index already exists.
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(false,None, [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-
- // batch deletion and addition
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
- }
-
- #[test]
- fn autobatch_primary_key() {
- // ==> If I have a pk
- // With a single update
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-
- // With a multiple updates
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-
- // ==> If I don't have a pk
- // With a single update
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-
- // With a multiple updates
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
- debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
- }
-}
diff --git a/crates/index-scheduler/src/batch.rs b/crates/index-scheduler/src/batch.rs
deleted file mode 100644
index 630471790..000000000
--- a/crates/index-scheduler/src/batch.rs
+++ /dev/null
@@ -1,1906 +0,0 @@
-/*!
-This module handles the creation and processing of batch operations.
-
-A batch is a combination of multiple tasks that can be processed at once.
-Executing a batch operation should always be functionally equivalent to
-executing each of its tasks' operations individually and in order.
-
-For example, if the user sends two tasks:
-1. import documents X
-2. import documents Y
-
-We can combine the two tasks in a single batch:
-1. import documents X and Y
-
-Processing this batch is functionally equivalent to processing the two
-tasks individually, but should be much faster since we are only performing
-one indexing operation.
-*/
-
-use std::collections::{BTreeSet, HashMap, HashSet};
-use std::ffi::OsStr;
-use std::fmt;
-use std::fs::{self, File};
-use std::io::BufWriter;
-use std::sync::atomic::{self, AtomicU64};
-use std::time::Duration;
-
-use bumpalo::collections::CollectIn;
-use bumpalo::Bump;
-use dump::IndexMetadata;
-use meilisearch_types::batches::BatchId;
-use meilisearch_types::error::Code;
-use meilisearch_types::heed::{RoTxn, RwTxn};
-use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader, PrimaryKey};
-use meilisearch_types::milli::heed::CompactionOption;
-use meilisearch_types::milli::update::new::indexer::{self, UpdateByFunction};
-use meilisearch_types::milli::update::{IndexDocumentsMethod, Settings as MilliSettings};
-use meilisearch_types::milli::vector::parsed_vectors::{
- ExplicitVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
-};
-use meilisearch_types::milli::{self, Filter, ThreadPoolNoAbortBuilder};
-use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
-use meilisearch_types::tasks::{
- Details, IndexSwap, Kind, KindWithContent, Status, Task, TaskProgress,
-};
-use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
-use roaring::RoaringBitmap;
-use time::macros::format_description;
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-use crate::autobatcher::{self, BatchKind};
-use crate::utils::{self, swap_index_uid_in_task, ProcessingBatch};
-use crate::{Error, IndexScheduler, Result, TaskId};
-
-/// Represents a combination of tasks that can all be processed at the same time.
-///
-/// A batch contains the set of tasks that it represents (accessible through
-/// [`self.ids()`](Batch::ids)), as well as additional information on how to
-/// be processed.
-#[derive(Debug)]
-pub(crate) enum Batch {
- TaskCancelation {
- /// The task cancelation itself.
- task: Task,
- },
- TaskDeletions(Vec),
- SnapshotCreation(Vec),
- Dump(Task),
- IndexOperation {
- op: IndexOperation,
- must_create_index: bool,
- },
- IndexCreation {
- index_uid: String,
- primary_key: Option,
- task: Task,
- },
- IndexUpdate {
- index_uid: String,
- primary_key: Option,
- task: Task,
- },
- IndexDeletion {
- index_uid: String,
- tasks: Vec,
- index_has_been_created: bool,
- },
- IndexSwap {
- task: Task,
- },
-}
-
-#[derive(Debug)]
-pub(crate) enum DocumentOperation {
- Add(Uuid),
- Delete(Vec),
-}
-
-/// A [batch](Batch) that combines multiple tasks operating on an index.
-#[derive(Debug)]
-pub(crate) enum IndexOperation {
- DocumentOperation {
- index_uid: String,
- primary_key: Option,
- method: IndexDocumentsMethod,
- documents_counts: Vec,
- operations: Vec,
- tasks: Vec,
- },
- DocumentEdition {
- index_uid: String,
- task: Task,
- },
- DocumentDeletion {
- index_uid: String,
- tasks: Vec,
- },
- DocumentClear {
- index_uid: String,
- tasks: Vec,
- },
- Settings {
- index_uid: String,
- // The boolean indicates if it's a settings deletion or creation.
- settings: Vec<(bool, Settings)>,
- tasks: Vec,
- },
- DocumentClearAndSetting {
- index_uid: String,
- cleared_tasks: Vec,
-
- // The boolean indicates if it's a settings deletion or creation.
- settings: Vec<(bool, Settings)>,
- settings_tasks: Vec,
- },
- SettingsAndDocumentOperation {
- index_uid: String,
-
- primary_key: Option,
- method: IndexDocumentsMethod,
- documents_counts: Vec,
- operations: Vec,
- document_import_tasks: Vec,
-
- // The boolean indicates if it's a settings deletion or creation.
- settings: Vec<(bool, Settings)>,
- settings_tasks: Vec,
- },
-}
-
-impl Batch {
- /// Return the task ids associated with this batch.
- pub fn ids(&self) -> RoaringBitmap {
- match self {
- Batch::TaskCancelation { task, .. }
- | Batch::Dump(task)
- | Batch::IndexCreation { task, .. }
- | Batch::IndexUpdate { task, .. } => {
- RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
- }
- Batch::SnapshotCreation(tasks)
- | Batch::TaskDeletions(tasks)
- | Batch::IndexDeletion { tasks, .. } => {
- RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
- }
- Batch::IndexOperation { op, .. } => match op {
- IndexOperation::DocumentOperation { tasks, .. }
- | IndexOperation::Settings { tasks, .. }
- | IndexOperation::DocumentDeletion { tasks, .. }
- | IndexOperation::DocumentClear { tasks, .. } => {
- RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
- }
- IndexOperation::DocumentEdition { task, .. } => {
- RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
- }
- IndexOperation::SettingsAndDocumentOperation {
- document_import_tasks: tasks,
- settings_tasks: other,
- ..
- }
- | IndexOperation::DocumentClearAndSetting {
- cleared_tasks: tasks,
- settings_tasks: other,
- ..
- } => RoaringBitmap::from_iter(tasks.iter().chain(other).map(|task| task.uid)),
- },
- Batch::IndexSwap { task } => {
- RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
- }
- }
- }
-
- /// Return the index UID associated with this batch
- pub fn index_uid(&self) -> Option<&str> {
- use Batch::*;
- match self {
- TaskCancelation { .. }
- | TaskDeletions(_)
- | SnapshotCreation(_)
- | Dump(_)
- | IndexSwap { .. } => None,
- IndexOperation { op, .. } => Some(op.index_uid()),
- IndexCreation { index_uid, .. }
- | IndexUpdate { index_uid, .. }
- | IndexDeletion { index_uid, .. } => Some(index_uid),
- }
- }
-}
-
-impl fmt::Display for Batch {
- /// A text used when we debug the profiling reports.
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let index_uid = self.index_uid();
- let tasks = self.ids();
- match self {
- Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
- Batch::TaskDeletions(_) => f.write_str("TaskDeletion")?,
- Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
- Batch::Dump(_) => f.write_str("Dump")?,
- Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
- Batch::IndexCreation { .. } => f.write_str("IndexCreation")?,
- Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
- Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
- Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
- };
- match index_uid {
- Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
- None => f.write_fmt(format_args!(" from tasks: {tasks:?}")),
- }
- }
-}
-
-impl IndexOperation {
- pub fn index_uid(&self) -> &str {
- match self {
- IndexOperation::DocumentOperation { index_uid, .. }
- | IndexOperation::DocumentEdition { index_uid, .. }
- | IndexOperation::DocumentDeletion { index_uid, .. }
- | IndexOperation::DocumentClear { index_uid, .. }
- | IndexOperation::Settings { index_uid, .. }
- | IndexOperation::DocumentClearAndSetting { index_uid, .. }
- | IndexOperation::SettingsAndDocumentOperation { index_uid, .. } => index_uid,
- }
- }
-}
-
-impl fmt::Display for IndexOperation {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self {
- IndexOperation::DocumentOperation { .. } => {
- f.write_str("IndexOperation::DocumentOperation")
- }
- IndexOperation::DocumentEdition { .. } => {
- f.write_str("IndexOperation::DocumentEdition")
- }
- IndexOperation::DocumentDeletion { .. } => {
- f.write_str("IndexOperation::DocumentDeletion")
- }
- IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
- IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
- IndexOperation::DocumentClearAndSetting { .. } => {
- f.write_str("IndexOperation::DocumentClearAndSetting")
- }
- IndexOperation::SettingsAndDocumentOperation { .. } => {
- f.write_str("IndexOperation::SettingsAndDocumentOperation")
- }
- }
- }
-}
-
-impl IndexScheduler {
- /// Convert an [`BatchKind`](crate::autobatcher::BatchKind) into a [`Batch`].
- ///
- /// ## Arguments
- /// - `rtxn`: read transaction
- /// - `index_uid`: name of the index affected by the operations of the autobatch
- /// - `batch`: the result of the autobatcher
- pub(crate) fn create_next_batch_index(
- &self,
- rtxn: &RoTxn,
- index_uid: String,
- batch: BatchKind,
- current_batch: &mut ProcessingBatch,
- must_create_index: bool,
- ) -> Result> {
- match batch {
- BatchKind::DocumentClear { ids } => Ok(Some(Batch::IndexOperation {
- op: IndexOperation::DocumentClear {
- tasks: self.get_existing_tasks_for_processing_batch(
- rtxn,
- current_batch,
- ids,
- )?,
- index_uid,
- },
- must_create_index,
- })),
- BatchKind::DocumentEdition { id } => {
- let mut task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- match &task.kind {
- KindWithContent::DocumentEdition { index_uid, .. } => {
- Ok(Some(Batch::IndexOperation {
- op: IndexOperation::DocumentEdition {
- index_uid: index_uid.clone(),
- task,
- },
- must_create_index: false,
- }))
- }
- _ => unreachable!(),
- }
- }
- BatchKind::DocumentOperation { method, operation_ids, .. } => {
- let tasks = self.get_existing_tasks_for_processing_batch(
- rtxn,
- current_batch,
- operation_ids,
- )?;
- let primary_key = tasks
- .iter()
- .find_map(|task| match task.kind {
- KindWithContent::DocumentAdditionOrUpdate { ref primary_key, .. } => {
- // we want to stop on the first document addition
- Some(primary_key.clone())
- }
- KindWithContent::DocumentDeletion { .. } => None,
- _ => unreachable!(),
- })
- .flatten();
-
- let mut documents_counts = Vec::new();
- let mut operations = Vec::new();
-
- for task in tasks.iter() {
- match task.kind {
- KindWithContent::DocumentAdditionOrUpdate {
- content_file,
- documents_count,
- ..
- } => {
- documents_counts.push(documents_count);
- operations.push(DocumentOperation::Add(content_file));
- }
- KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
- documents_counts.push(documents_ids.len() as u64);
- operations.push(DocumentOperation::Delete(documents_ids.clone()));
- }
- _ => unreachable!(),
- }
- }
-
- Ok(Some(Batch::IndexOperation {
- op: IndexOperation::DocumentOperation {
- index_uid,
- primary_key,
- method,
- documents_counts,
- operations,
- tasks,
- },
- must_create_index,
- }))
- }
- BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: _ } => {
- let tasks = self.get_existing_tasks_for_processing_batch(
- rtxn,
- current_batch,
- deletion_ids,
- )?;
-
- Ok(Some(Batch::IndexOperation {
- op: IndexOperation::DocumentDeletion { index_uid, tasks },
- must_create_index,
- }))
- }
- BatchKind::Settings { settings_ids, .. } => {
- let tasks = self.get_existing_tasks_for_processing_batch(
- rtxn,
- current_batch,
- settings_ids,
- )?;
-
- let mut settings = Vec::new();
- for task in &tasks {
- match task.kind {
- KindWithContent::SettingsUpdate {
- ref new_settings, is_deletion, ..
- } => settings.push((is_deletion, *new_settings.clone())),
- _ => unreachable!(),
- }
- }
-
- Ok(Some(Batch::IndexOperation {
- op: IndexOperation::Settings { index_uid, settings, tasks },
- must_create_index,
- }))
- }
- BatchKind::ClearAndSettings { other, settings_ids, allow_index_creation } => {
- let (index_uid, settings, settings_tasks) = match self
- .create_next_batch_index(
- rtxn,
- index_uid,
- BatchKind::Settings { settings_ids, allow_index_creation },
- current_batch,
- must_create_index,
- )?
- .unwrap()
- {
- Batch::IndexOperation {
- op: IndexOperation::Settings { index_uid, settings, tasks, .. },
- ..
- } => (index_uid, settings, tasks),
- _ => unreachable!(),
- };
- let (index_uid, cleared_tasks) = match self
- .create_next_batch_index(
- rtxn,
- index_uid,
- BatchKind::DocumentClear { ids: other },
- current_batch,
- must_create_index,
- )?
- .unwrap()
- {
- Batch::IndexOperation {
- op: IndexOperation::DocumentClear { index_uid, tasks },
- ..
- } => (index_uid, tasks),
- _ => unreachable!(),
- };
-
- Ok(Some(Batch::IndexOperation {
- op: IndexOperation::DocumentClearAndSetting {
- index_uid,
- cleared_tasks,
- settings,
- settings_tasks,
- },
- must_create_index,
- }))
- }
- BatchKind::SettingsAndDocumentOperation {
- settings_ids,
- method,
- allow_index_creation,
- primary_key,
- operation_ids,
- } => {
- let settings = self.create_next_batch_index(
- rtxn,
- index_uid.clone(),
- BatchKind::Settings { settings_ids, allow_index_creation },
- current_batch,
- must_create_index,
- )?;
-
- let document_import = self.create_next_batch_index(
- rtxn,
- index_uid.clone(),
- BatchKind::DocumentOperation {
- method,
- allow_index_creation,
- primary_key,
- operation_ids,
- },
- current_batch,
- must_create_index,
- )?;
-
- match (document_import, settings) {
- (
- Some(Batch::IndexOperation {
- op:
- IndexOperation::DocumentOperation {
- primary_key,
- documents_counts,
- operations,
- tasks: document_import_tasks,
- ..
- },
- ..
- }),
- Some(Batch::IndexOperation {
- op: IndexOperation::Settings { settings, tasks: settings_tasks, .. },
- ..
- }),
- ) => Ok(Some(Batch::IndexOperation {
- op: IndexOperation::SettingsAndDocumentOperation {
- index_uid,
- primary_key,
- method,
- documents_counts,
- operations,
- document_import_tasks,
- settings,
- settings_tasks,
- },
- must_create_index,
- })),
- _ => unreachable!(),
- }
- }
- BatchKind::IndexCreation { id } => {
- let mut task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- let (index_uid, primary_key) = match &task.kind {
- KindWithContent::IndexCreation { index_uid, primary_key } => {
- (index_uid.clone(), primary_key.clone())
- }
- _ => unreachable!(),
- };
- Ok(Some(Batch::IndexCreation { index_uid, primary_key, task }))
- }
- BatchKind::IndexUpdate { id } => {
- let mut task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- let primary_key = match &task.kind {
- KindWithContent::IndexUpdate { primary_key, .. } => primary_key.clone(),
- _ => unreachable!(),
- };
- Ok(Some(Batch::IndexUpdate { index_uid, primary_key, task }))
- }
- BatchKind::IndexDeletion { ids } => Ok(Some(Batch::IndexDeletion {
- index_uid,
- index_has_been_created: must_create_index,
- tasks: self.get_existing_tasks_for_processing_batch(rtxn, current_batch, ids)?,
- })),
- BatchKind::IndexSwap { id } => {
- let mut task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- Ok(Some(Batch::IndexSwap { task }))
- }
- }
- }
-
- /// Create the next batch to be processed;
- /// 1. We get the *last* task to cancel.
- /// 2. We get the *next* task to delete.
- /// 3. We get the *next* snapshot to process.
- /// 4. We get the *next* dump to process.
- /// 5. We get the *next* tasks to process for a specific index.
- #[tracing::instrument(level = "trace", skip(self, rtxn), target = "indexing::scheduler")]
- pub(crate) fn create_next_batch(
- &self,
- rtxn: &RoTxn,
- ) -> Result> {
- #[cfg(test)]
- self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
-
- let batch_id = self.next_batch_id(rtxn)?;
- let mut current_batch = ProcessingBatch::new(batch_id);
-
- let enqueued = &self.get_status(rtxn, Status::Enqueued)?;
- let to_cancel = self.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
-
- // 1. we get the last task to cancel.
- if let Some(task_id) = to_cancel.max() {
- let mut task = self.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- return Ok(Some((Batch::TaskCancelation { task }, current_batch)));
- }
-
- // 2. we get the next task to delete
- let to_delete = self.get_kind(rtxn, Kind::TaskDeletion)? & enqueued;
- if !to_delete.is_empty() {
- let mut tasks = self.get_existing_tasks(rtxn, to_delete)?;
- current_batch.processing(&mut tasks);
- return Ok(Some((Batch::TaskDeletions(tasks), current_batch)));
- }
-
- // 3. we batch the snapshot.
- let to_snapshot = self.get_kind(rtxn, Kind::SnapshotCreation)? & enqueued;
- if !to_snapshot.is_empty() {
- let mut tasks = self.get_existing_tasks(rtxn, to_snapshot)?;
- current_batch.processing(&mut tasks);
- return Ok(Some((Batch::SnapshotCreation(tasks), current_batch)));
- }
-
- // 4. we batch the dumps.
- let to_dump = self.get_kind(rtxn, Kind::DumpCreation)? & enqueued;
- if let Some(to_dump) = to_dump.min() {
- let mut task = self.get_task(rtxn, to_dump)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
- return Ok(Some((Batch::Dump(task), current_batch)));
- }
-
- // 5. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
- let task_id = if let Some(task_id) = enqueued.min() { task_id } else { return Ok(None) };
- let mut task = self.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
- current_batch.processing(Some(&mut task));
-
- // If the task is not associated with any index, verify that it is an index swap and
- // create the batch directly. Otherwise, get the index name associated with the task
- // and use the autobatcher to batch the enqueued tasks associated with it
-
- let index_name = if let Some(&index_name) = task.indexes().first() {
- index_name
- } else {
- assert!(matches!(&task.kind, KindWithContent::IndexSwap { swaps } if swaps.is_empty()));
- return Ok(Some((Batch::IndexSwap { task }, current_batch)));
- };
-
- let index_already_exists = self.index_mapper.exists(rtxn, index_name)?;
- let mut primary_key = None;
- if index_already_exists {
- let index = self.index_mapper.index(rtxn, index_name)?;
- let rtxn = index.read_txn()?;
- primary_key = index.primary_key(&rtxn)?.map(|pk| pk.to_string());
- }
-
- let index_tasks = self.index_tasks(rtxn, index_name)? & enqueued;
-
- // If autobatching is disabled we only take one task at a time.
- // Otherwise, we take only a maximum of tasks to create batches.
- let tasks_limit =
- if self.autobatching_enabled { self.max_number_of_batched_tasks } else { 1 };
-
- let enqueued = index_tasks
- .into_iter()
- .take(tasks_limit)
- .map(|task_id| {
- self.get_task(rtxn, task_id)
- .and_then(|task| task.ok_or(Error::CorruptedTaskQueue))
- .map(|task| (task.uid, task.kind))
- })
- .collect::>>()?;
-
- if let Some((batchkind, create_index)) =
- autobatcher::autobatch(enqueued, index_already_exists, primary_key.as_deref())
- {
- return Ok(self
- .create_next_batch_index(
- rtxn,
- index_name.to_string(),
- batchkind,
- &mut current_batch,
- create_index,
- )?
- .map(|batch| (batch, current_batch)));
- }
-
- // If we found no tasks then we were notified for something that got autobatched
- // somehow and there is nothing to do.
- Ok(None)
- }
-
- /// Apply the operation associated with the given batch.
- ///
- /// ## Return
- /// The list of tasks that were processed. The metadata of each task in the returned
- /// list is updated accordingly, with the exception of the its date fields
- /// [`finished_at`](meilisearch_types::tasks::Task::finished_at) and [`started_at`](meilisearch_types::tasks::Task::started_at).
- #[tracing::instrument(level = "trace", skip(self, batch), target = "indexing::scheduler", fields(batch=batch.to_string()))]
- pub(crate) fn process_batch(
- &self,
- batch: Batch,
- current_batch: &mut ProcessingBatch,
- ) -> Result> {
- #[cfg(test)]
- {
- self.maybe_fail(crate::tests::FailureLocation::InsideProcessBatch)?;
- self.maybe_fail(crate::tests::FailureLocation::PanicInsideProcessBatch)?;
- self.breakpoint(crate::Breakpoint::InsideProcessBatch);
- }
-
- match batch {
- Batch::TaskCancelation { mut task } => {
- // 1. Retrieve the tasks that matched the query at enqueue-time.
- let matched_tasks =
- if let KindWithContent::TaskCancelation { tasks, query: _ } = &task.kind {
- tasks
- } else {
- unreachable!()
- };
-
- let rtxn = self.env.read_txn()?;
- let mut canceled_tasks =
- self.cancel_matched_tasks(&rtxn, task.uid, current_batch, matched_tasks)?;
-
- task.status = Status::Succeeded;
- match &mut task.details {
- Some(Details::TaskCancelation {
- matched_tasks: _,
- canceled_tasks: canceled_tasks_details,
- original_filter: _,
- }) => {
- *canceled_tasks_details = Some(canceled_tasks.len() as u64);
- }
- _ => unreachable!(),
- }
-
- canceled_tasks.push(task);
-
- Ok(canceled_tasks)
- }
- Batch::TaskDeletions(mut tasks) => {
- // 1. Retrieve the tasks that matched the query at enqueue-time.
- let mut matched_tasks = RoaringBitmap::new();
-
- for task in tasks.iter() {
- if let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind {
- matched_tasks |= tasks;
- } else {
- unreachable!()
- }
- }
-
- let mut wtxn = self.env.write_txn()?;
- let mut deleted_tasks = self.delete_matched_tasks(&mut wtxn, &matched_tasks)?;
- wtxn.commit()?;
-
- for task in tasks.iter_mut() {
- task.status = Status::Succeeded;
- let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind else {
- unreachable!()
- };
-
- let deleted_tasks_count = deleted_tasks.intersection_len(tasks);
- deleted_tasks -= tasks;
-
- match &mut task.details {
- Some(Details::TaskDeletion {
- matched_tasks: _,
- deleted_tasks,
- original_filter: _,
- }) => {
- *deleted_tasks = Some(deleted_tasks_count);
- }
- _ => unreachable!(),
- }
- }
- Ok(tasks)
- }
- Batch::SnapshotCreation(mut tasks) => {
- fs::create_dir_all(&self.snapshots_path)?;
- let temp_snapshot_dir = tempfile::tempdir()?;
-
- // 1. Snapshot the version file.
- let dst = temp_snapshot_dir.path().join(VERSION_FILE_NAME);
- fs::copy(&self.version_file_path, dst)?;
-
- // 2. Snapshot the index-scheduler LMDB env
- //
- // When we call copy_to_file, LMDB opens a read transaction by itself,
- // we can't provide our own. It is an issue as we would like to know
- // the update files to copy but new ones can be enqueued between the copy
- // of the env and the new transaction we open to retrieve the enqueued tasks.
- // So we prefer opening a new transaction after copying the env and copy more
- // update files than not enough.
- //
- // Note that there cannot be any update files deleted between those
- // two read operations as the task processing is synchronous.
-
- // 2.1 First copy the LMDB env of the index-scheduler
- let dst = temp_snapshot_dir.path().join("tasks");
- fs::create_dir_all(&dst)?;
- self.env.copy_to_file(dst.join("data.mdb"), CompactionOption::Enabled)?;
-
- // 2.2 Create a read transaction on the index-scheduler
- let rtxn = self.env.read_txn()?;
-
- // 2.3 Create the update files directory
- let update_files_dir = temp_snapshot_dir.path().join("update_files");
- fs::create_dir_all(&update_files_dir)?;
-
- // 2.4 Only copy the update files of the enqueued tasks
- for task_id in self.get_status(&rtxn, Status::Enqueued)? {
- let task = self.get_task(&rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
- if let Some(content_uuid) = task.content_uuid() {
- let src = self.file_store.get_update_path(content_uuid);
- let dst = update_files_dir.join(content_uuid.to_string());
- fs::copy(src, dst)?;
- }
- }
-
- // 3. Snapshot every indexes
- for result in self.index_mapper.index_mapping.iter(&rtxn)? {
- let (name, uuid) = result?;
- let index = self.index_mapper.index(&rtxn, name)?;
- let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string());
- fs::create_dir_all(&dst)?;
- index.copy_to_file(dst.join("data.mdb"), CompactionOption::Enabled)?;
- }
-
- drop(rtxn);
-
- // 4. Snapshot the auth LMDB env
- let dst = temp_snapshot_dir.path().join("auth");
- fs::create_dir_all(&dst)?;
- // TODO We can't use the open_auth_store_env function here but we should
- let auth = unsafe {
- milli::heed::EnvOpenOptions::new()
- .map_size(1024 * 1024 * 1024) // 1 GiB
- .max_dbs(2)
- .open(&self.auth_path)
- }?;
- auth.copy_to_file(dst.join("data.mdb"), CompactionOption::Enabled)?;
-
- // 5. Copy and tarball the flat snapshot
- // 5.1 Find the original name of the database
- // TODO find a better way to get this path
- let mut base_path = self.env.path().to_owned();
- base_path.pop();
- let db_name = base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms");
-
- // 5.2 Tarball the content of the snapshot in a tempfile with a .snapshot extension
- let snapshot_path = self.snapshots_path.join(format!("{}.snapshot", db_name));
- let temp_snapshot_file = tempfile::NamedTempFile::new_in(&self.snapshots_path)?;
- compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path())?;
- let file = temp_snapshot_file.persist(snapshot_path)?;
-
- // 5.3 Change the permission to make the snapshot readonly
- let mut permissions = file.metadata()?.permissions();
- permissions.set_readonly(true);
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt;
- #[allow(clippy::non_octal_unix_permissions)]
- // rwxrwxrwx
- permissions.set_mode(0b100100100);
- }
-
- file.set_permissions(permissions)?;
-
- for task in &mut tasks {
- task.status = Status::Succeeded;
- }
-
- Ok(tasks)
- }
- Batch::Dump(mut task) => {
- let started_at = OffsetDateTime::now_utc();
- let (keys, instance_uid) =
- if let KindWithContent::DumpCreation { keys, instance_uid } = &task.kind {
- (keys, instance_uid)
- } else {
- unreachable!();
- };
- let dump = dump::DumpWriter::new(*instance_uid)?;
-
- // 1. dump the keys
- let mut dump_keys = dump.create_keys()?;
- for key in keys {
- dump_keys.push_key(key)?;
- }
- dump_keys.flush()?;
-
- let rtxn = self.env.read_txn()?;
-
- // 2. dump the tasks
- let mut dump_tasks = dump.create_tasks_queue()?;
- for ret in self.all_tasks.iter(&rtxn)? {
- if self.must_stop_processing.get() {
- return Err(Error::AbortedTask);
- }
-
- let (_, mut t) = ret?;
- let status = t.status;
- let content_file = t.content_uuid();
-
- // In the case we're dumping ourselves we want to be marked as finished
- // to not loop over ourselves indefinitely.
- if t.uid == task.uid {
- let finished_at = OffsetDateTime::now_utc();
-
- // We're going to fake the date because we don't know if everything is going to go well.
- // But we need to dump the task as finished and successful.
- // If something fail everything will be set appropriately in the end.
- t.status = Status::Succeeded;
- t.started_at = Some(started_at);
- t.finished_at = Some(finished_at);
- }
- let mut dump_content_file = dump_tasks.push_task(&t.into())?;
-
- // 2.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet.
- if let Some(content_file) = content_file {
- if self.must_stop_processing.get() {
- return Err(Error::AbortedTask);
- }
- if status == Status::Enqueued {
- let content_file = self.file_store.get_update(content_file)?;
-
- let reader = DocumentsBatchReader::from_reader(content_file)
- .map_err(milli::Error::from)?;
-
- let (mut cursor, documents_batch_index) =
- reader.into_cursor_and_fields_index();
-
- while let Some(doc) =
- cursor.next_document().map_err(milli::Error::from)?
- {
- dump_content_file
- .push_document(&obkv_to_object(doc, &documents_batch_index)?)?;
- }
- dump_content_file.flush()?;
- }
- }
- }
- dump_tasks.flush()?;
-
- // 3. Dump the indexes
- self.index_mapper.try_for_each_index(&rtxn, |uid, index| -> Result<()> {
- let rtxn = index.read_txn()?;
- let metadata = IndexMetadata {
- uid: uid.to_owned(),
- primary_key: index.primary_key(&rtxn)?.map(String::from),
- created_at: index.created_at(&rtxn)?,
- updated_at: index.updated_at(&rtxn)?,
- };
- let mut index_dumper = dump.create_index(uid, &metadata)?;
-
- let fields_ids_map = index.fields_ids_map(&rtxn)?;
- let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
- let embedding_configs = index.embedding_configs(&rtxn)?;
-
- // 3.1. Dump the documents
- for ret in index.all_documents(&rtxn)? {
- if self.must_stop_processing.get() {
- return Err(Error::AbortedTask);
- }
-
- let (id, doc) = ret?;
-
- let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
-
- 'inject_vectors: {
- let embeddings = index.embeddings(&rtxn, id)?;
-
- if embeddings.is_empty() {
- break 'inject_vectors;
- }
-
- let vectors = document
- .entry(RESERVED_VECTORS_FIELD_NAME.to_owned())
- .or_insert(serde_json::Value::Object(Default::default()));
-
- let serde_json::Value::Object(vectors) = vectors else {
- return Err(milli::Error::UserError(
- milli::UserError::InvalidVectorsMapType {
- document_id: {
- if let Ok(Some(Ok(index))) = index
- .external_id_of(&rtxn, std::iter::once(id))
- .map(|it| it.into_iter().next())
- {
- index
- } else {
- format!("internal docid={id}")
- }
- },
- value: vectors.clone(),
- },
- )
- .into());
- };
-
- for (embedder_name, embeddings) in embeddings {
- let user_provided = embedding_configs
- .iter()
- .find(|conf| conf.name == embedder_name)
- .is_some_and(|conf| conf.user_provided.contains(id));
-
- let embeddings = ExplicitVectors {
- embeddings: Some(
- VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
- ),
- regenerate: !user_provided,
- };
- vectors.insert(
- embedder_name,
- serde_json::to_value(embeddings).unwrap(),
- );
- }
- }
-
- index_dumper.push_document(&document)?;
- }
-
- // 3.2. Dump the settings
- let settings = meilisearch_types::settings::settings(
- index,
- &rtxn,
- meilisearch_types::settings::SecretPolicy::RevealSecrets,
- )?;
- index_dumper.settings(&settings)?;
- Ok(())
- })?;
-
- // 4. Dump experimental feature settings
- let features = self.features().runtime_features();
- dump.create_experimental_features(features)?;
-
- let dump_uid = started_at.format(format_description!(
- "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
- )).unwrap();
-
- if self.must_stop_processing.get() {
- return Err(Error::AbortedTask);
- }
- let path = self.dumps_path.join(format!("{}.dump", dump_uid));
- let file = File::create(path)?;
- dump.persist_to(BufWriter::new(file))?;
-
- // if we reached this step we can tell the scheduler we succeeded to dump ourselves.
- task.status = Status::Succeeded;
- task.details = Some(Details::Dump { dump_uid: Some(dump_uid) });
- Ok(vec![task])
- }
- Batch::IndexOperation { op, must_create_index } => {
- let index_uid = op.index_uid().to_string();
- let index = if must_create_index {
- // create the index if it doesn't already exist
- let wtxn = self.env.write_txn()?;
- self.index_mapper.create_index(wtxn, &index_uid, None)?
- } else {
- let rtxn = self.env.read_txn()?;
- self.index_mapper.index(&rtxn, &index_uid)?
- };
-
- // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick
- self.index_mapper
- .set_currently_updating_index(Some((index_uid.clone(), index.clone())));
-
- let mut index_wtxn = index.write_txn()?;
- let tasks = self.apply_index_operation(&mut index_wtxn, &index, op)?;
- index_wtxn.commit()?;
-
- // if the update processed successfully, we're going to store the new
- // stats of the index. Since the tasks have already been processed and
- // this is a non-critical operation. If it fails, we should not fail
- // the entire batch.
- let res = || -> Result<()> {
- let index_rtxn = index.read_txn()?;
- let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)?;
- let mut wtxn = self.env.write_txn()?;
- self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?;
- wtxn.commit()?;
- Ok(())
- }();
-
- match res {
- Ok(_) => (),
- Err(e) => tracing::error!(
- error = &e as &dyn std::error::Error,
- "Could not write the stats of the index"
- ),
- }
-
- Ok(tasks)
- }
- Batch::IndexCreation { index_uid, primary_key, task } => {
- let wtxn = self.env.write_txn()?;
- if self.index_mapper.exists(&wtxn, &index_uid)? {
- return Err(Error::IndexAlreadyExists(index_uid));
- }
- self.index_mapper.create_index(wtxn, &index_uid, None)?;
-
- self.process_batch(
- Batch::IndexUpdate { index_uid, primary_key, task },
- current_batch,
- )
- }
- Batch::IndexUpdate { index_uid, primary_key, mut task } => {
- let rtxn = self.env.read_txn()?;
- let index = self.index_mapper.index(&rtxn, &index_uid)?;
-
- if let Some(primary_key) = primary_key.clone() {
- let mut index_wtxn = index.write_txn()?;
- let mut builder = MilliSettings::new(
- &mut index_wtxn,
- &index,
- self.index_mapper.indexer_config(),
- );
- builder.set_primary_key(primary_key);
- let must_stop_processing = self.must_stop_processing.clone();
- builder.execute(
- |indexing_step| tracing::debug!(update = ?indexing_step),
- || must_stop_processing.get(),
- )?;
- index_wtxn.commit()?;
- }
-
- // drop rtxn before starting a new wtxn on the same db
- rtxn.commit()?;
-
- task.status = Status::Succeeded;
- task.details = Some(Details::IndexInfo { primary_key });
-
- // if the update processed successfully, we're going to store the new
- // stats of the index. Since the tasks have already been processed and
- // this is a non-critical operation. If it fails, we should not fail
- // the entire batch.
- let res = || -> Result<()> {
- let mut wtxn = self.env.write_txn()?;
- let index_rtxn = index.read_txn()?;
- let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)?;
- self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?;
- wtxn.commit()?;
- Ok(())
- }();
-
- match res {
- Ok(_) => (),
- Err(e) => tracing::error!(
- error = &e as &dyn std::error::Error,
- "Could not write the stats of the index"
- ),
- }
-
- Ok(vec![task])
- }
- Batch::IndexDeletion { index_uid, index_has_been_created, mut tasks } => {
- let wtxn = self.env.write_txn()?;
-
- // it's possible that the index doesn't exist
- let number_of_documents = || -> Result {
- let index = self.index_mapper.index(&wtxn, &index_uid)?;
- let index_rtxn = index.read_txn()?;
- Ok(index.number_of_documents(&index_rtxn)?)
- }()
- .unwrap_or_default();
-
- // The write transaction is directly owned and committed inside.
- match self.index_mapper.delete_index(wtxn, &index_uid) {
- Ok(()) => (),
- Err(Error::IndexNotFound(_)) if index_has_been_created => (),
- Err(e) => return Err(e),
- }
-
- // We set all the tasks details to the default value.
- for task in &mut tasks {
- task.status = Status::Succeeded;
- task.details = match &task.kind {
- KindWithContent::IndexDeletion { .. } => {
- Some(Details::ClearAll { deleted_documents: Some(number_of_documents) })
- }
- otherwise => otherwise.default_finished_details(),
- };
- }
-
- Ok(tasks)
- }
- Batch::IndexSwap { mut task } => {
- let mut wtxn = self.env.write_txn()?;
- let swaps = if let KindWithContent::IndexSwap { swaps } = &task.kind {
- swaps
- } else {
- unreachable!()
- };
- let mut not_found_indexes = BTreeSet::new();
- for IndexSwap { indexes: (lhs, rhs) } in swaps {
- for index in [lhs, rhs] {
- let index_exists = self.index_mapper.index_exists(&wtxn, index)?;
- if !index_exists {
- not_found_indexes.insert(index);
- }
- }
- }
- if !not_found_indexes.is_empty() {
- if not_found_indexes.len() == 1 {
- return Err(Error::SwapIndexNotFound(
- not_found_indexes.into_iter().next().unwrap().clone(),
- ));
- } else {
- return Err(Error::SwapIndexesNotFound(
- not_found_indexes.into_iter().cloned().collect(),
- ));
- }
- }
- for swap in swaps {
- self.apply_index_swap(&mut wtxn, task.uid, &swap.indexes.0, &swap.indexes.1)?;
- }
- wtxn.commit()?;
- task.status = Status::Succeeded;
- Ok(vec![task])
- }
- }
- }
-
- /// Swap the index `lhs` with the index `rhs`.
- fn apply_index_swap(&self, wtxn: &mut RwTxn, task_id: u32, lhs: &str, rhs: &str) -> Result<()> {
- // 1. Verify that both lhs and rhs are existing indexes
- let index_lhs_exists = self.index_mapper.index_exists(wtxn, lhs)?;
- if !index_lhs_exists {
- return Err(Error::IndexNotFound(lhs.to_owned()));
- }
- let index_rhs_exists = self.index_mapper.index_exists(wtxn, rhs)?;
- if !index_rhs_exists {
- return Err(Error::IndexNotFound(rhs.to_owned()));
- }
-
- // 2. Get the task set for index = name that appeared before the index swap task
- let mut index_lhs_task_ids = self.index_tasks(wtxn, lhs)?;
- index_lhs_task_ids.remove_range(task_id..);
- let mut index_rhs_task_ids = self.index_tasks(wtxn, rhs)?;
- index_rhs_task_ids.remove_range(task_id..);
-
- // 3. before_name -> new_name in the task's KindWithContent
- for task_id in &index_lhs_task_ids | &index_rhs_task_ids {
- let mut task = self.get_task(wtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
- swap_index_uid_in_task(&mut task, (lhs, rhs));
- self.all_tasks.put(wtxn, &task_id, &task)?;
- }
-
- // 4. remove the task from indexuid = before_name
- // 5. add the task to indexuid = after_name
- self.update_index(wtxn, lhs, |lhs_tasks| {
- *lhs_tasks -= &index_lhs_task_ids;
- *lhs_tasks |= &index_rhs_task_ids;
- })?;
- self.update_index(wtxn, rhs, |rhs_tasks| {
- *rhs_tasks -= &index_rhs_task_ids;
- *rhs_tasks |= &index_lhs_task_ids;
- })?;
-
- // 6. Swap in the index mapper
- self.index_mapper.swap(wtxn, lhs, rhs)?;
-
- Ok(())
- }
-
- /// Process the index operation on the given index.
- ///
- /// ## Return
- /// The list of processed tasks.
- #[tracing::instrument(
- level = "trace",
- skip(self, index_wtxn, index),
- target = "indexing::scheduler"
- )]
- fn apply_index_operation<'i>(
- &self,
- index_wtxn: &mut RwTxn<'i>,
- index: &'i Index,
- operation: IndexOperation,
- ) -> Result> {
- let indexer_alloc = Bump::new();
-
- let started_processing_at = std::time::Instant::now();
- let secs_since_started_processing_at = AtomicU64::new(0);
- const PRINT_SECS_DELTA: u64 = 5;
-
- let processing_tasks = self.processing_tasks.clone();
- let must_stop_processing = self.must_stop_processing.clone();
- let send_progress = |progress| {
- let now = std::time::Instant::now();
- let elapsed = secs_since_started_processing_at.load(atomic::Ordering::Relaxed);
- let previous = started_processing_at + Duration::from_secs(elapsed);
- let elapsed = now - previous;
-
- if elapsed.as_secs() < PRINT_SECS_DELTA {
- return;
- }
-
- secs_since_started_processing_at
- .store((now - started_processing_at).as_secs(), atomic::Ordering::Relaxed);
-
- let TaskProgress {
- current_step,
- finished_steps,
- total_steps,
- finished_substeps,
- total_substeps,
- } = processing_tasks.write().unwrap().update_progress(progress);
-
- tracing::info!(
- current_step,
- finished_steps,
- total_steps,
- finished_substeps,
- total_substeps
- );
- };
-
- match operation {
- IndexOperation::DocumentClear { mut tasks, .. } => {
- let count = milli::update::ClearDocuments::new(index_wtxn, index).execute()?;
-
- let mut first_clear_found = false;
- for task in &mut tasks {
- task.status = Status::Succeeded;
- // The first document clear will effectively delete every documents
- // in the database but the next ones will clear 0 documents.
- task.details = match &task.kind {
- KindWithContent::DocumentClear { .. } => {
- let count = if first_clear_found { 0 } else { count };
- first_clear_found = true;
- Some(Details::ClearAll { deleted_documents: Some(count) })
- }
- otherwise => otherwise.default_details(),
- };
- }
-
- Ok(tasks)
- }
- IndexOperation::DocumentOperation {
- index_uid: _,
- primary_key,
- method,
- documents_counts: _,
- operations,
- mut tasks,
- } => {
- // TODO: at some point, for better efficiency we might want to reuse the bumpalo for successive batches.
- // this is made difficult by the fact we're doing private clones of the index scheduler and sending it
- // to a fresh thread.
- let mut content_files = Vec::new();
- for operation in &operations {
- if let DocumentOperation::Add(content_uuid) = operation {
- let content_file = self.file_store.get_update(*content_uuid)?;
- let mmap = unsafe { memmap2::Mmap::map(&content_file)? };
- if !mmap.is_empty() {
- content_files.push(mmap);
- }
- }
- }
-
- let rtxn = index.read_txn()?;
- let db_fields_ids_map = index.fields_ids_map(&rtxn)?;
- let mut new_fields_ids_map = db_fields_ids_map.clone();
-
- let mut content_files_iter = content_files.iter();
- let mut indexer = indexer::DocumentOperation::new(method);
- let embedders = index.embedding_configs(index_wtxn)?;
- let embedders = self.embedders(embedders)?;
- for operation in operations {
- match operation {
- DocumentOperation::Add(_content_uuid) => {
- let mmap = content_files_iter.next().unwrap();
- indexer.add_documents(mmap)?;
- }
- DocumentOperation::Delete(document_ids) => {
- let document_ids: bumpalo::collections::vec::Vec<_> = document_ids
- .iter()
- .map(|s| &*indexer_alloc.alloc_str(s))
- .collect_in(&indexer_alloc);
- indexer.delete_documents(document_ids.into_bump_slice());
- }
- }
- }
-
- let local_pool;
- let indexer_config = self.index_mapper.indexer_config();
- let pool = match &indexer_config.thread_pool {
- Some(pool) => pool,
- None => {
- local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
- &local_pool
- }
- };
-
- let (document_changes, operation_stats, primary_key) = indexer.into_changes(
- &indexer_alloc,
- index,
- &rtxn,
- primary_key.as_deref(),
- &mut new_fields_ids_map,
- &|| must_stop_processing.get(),
- &send_progress,
- )?;
-
- let mut addition = 0;
- for (stats, task) in operation_stats.into_iter().zip(&mut tasks) {
- addition += stats.document_count;
- match stats.error {
- Some(error) => {
- task.status = Status::Failed;
- task.error = Some(milli::Error::UserError(error).into());
- }
- None => task.status = Status::Succeeded,
- }
-
- task.details = match task.details {
- Some(Details::DocumentAdditionOrUpdate { received_documents, .. }) => {
- Some(Details::DocumentAdditionOrUpdate {
- received_documents,
- indexed_documents: Some(stats.document_count),
- })
- }
- Some(Details::DocumentDeletion { provided_ids, .. }) => {
- Some(Details::DocumentDeletion {
- provided_ids,
- deleted_documents: Some(stats.document_count),
- })
- }
- _ => {
- // In the case of a `documentAdditionOrUpdate` or `DocumentDeletion`
- // the details MUST be set to either addition or deletion
- unreachable!();
- }
- }
- }
-
- if tasks.iter().any(|res| res.error.is_none()) {
- pool.install(|| {
- indexer::index(
- index_wtxn,
- index,
- indexer_config.grenad_parameters(),
- &db_fields_ids_map,
- new_fields_ids_map,
- primary_key,
- &document_changes,
- embedders,
- &|| must_stop_processing.get(),
- &send_progress,
- )
- })
- .unwrap()?;
-
- tracing::info!(indexing_result = ?addition, processed_in = ?started_processing_at.elapsed(), "document indexing done");
- }
-
- Ok(tasks)
- }
- IndexOperation::DocumentEdition { mut task, .. } => {
- let (filter, code) = if let KindWithContent::DocumentEdition {
- filter_expr,
- context: _,
- function,
- ..
- } = &task.kind
- {
- (filter_expr, function)
- } else {
- unreachable!()
- };
-
- let candidates = match filter.as_ref().map(Filter::from_json) {
- Some(Ok(Some(filter))) => {
- filter.evaluate(index_wtxn, index).map_err(|err| match err {
- milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
- Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
- }
- e => e.into(),
- })?
- }
- None | Some(Ok(None)) => index.documents_ids(index_wtxn)?,
- Some(Err(e)) => return Err(e.into()),
- };
-
- let (original_filter, context, function) = if let Some(Details::DocumentEdition {
- original_filter,
- context,
- function,
- ..
- }) = task.details
- {
- (original_filter, context, function)
- } else {
- // In the case of a `documentEdition` the details MUST be set
- unreachable!();
- };
-
- if candidates.is_empty() {
- task.status = Status::Succeeded;
- task.details = Some(Details::DocumentEdition {
- original_filter,
- context,
- function,
- deleted_documents: Some(0),
- edited_documents: Some(0),
- });
-
- return Ok(vec![task]);
- }
-
- let rtxn = index.read_txn()?;
- let db_fields_ids_map = index.fields_ids_map(&rtxn)?;
- let mut new_fields_ids_map = db_fields_ids_map.clone();
- // candidates not empty => index not empty => a primary key is set
- let primary_key = index.primary_key(&rtxn)?.unwrap();
-
- let primary_key = PrimaryKey::new_or_insert(primary_key, &mut new_fields_ids_map)
- .map_err(milli::Error::from)?;
-
- let result_count = Ok((candidates.len(), candidates.len())) as Result<_>;
-
- if task.error.is_none() {
- let local_pool;
- let indexer_config = self.index_mapper.indexer_config();
- let pool = match &indexer_config.thread_pool {
- Some(pool) => pool,
- None => {
- local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
- &local_pool
- }
- };
-
- pool.install(|| {
- let indexer =
- UpdateByFunction::new(candidates, context.clone(), code.clone());
- let document_changes = indexer.into_changes(&primary_key)?;
- let embedders = index.embedding_configs(index_wtxn)?;
- let embedders = self.embedders(embedders)?;
-
- indexer::index(
- index_wtxn,
- index,
- indexer_config.grenad_parameters(),
- &db_fields_ids_map,
- new_fields_ids_map,
- None, // cannot change primary key in DocumentEdition
- &document_changes,
- embedders,
- &|| must_stop_processing.get(),
- &send_progress,
- )?;
-
- Result::Ok(())
- })
- .unwrap()?;
-
- // tracing::info!(indexing_result = ?addition, processed_in = ?started_processing_at.elapsed(), "document indexing done");
- }
-
- match result_count {
- Ok((deleted_documents, edited_documents)) => {
- task.status = Status::Succeeded;
- task.details = Some(Details::DocumentEdition {
- original_filter,
- context,
- function,
- deleted_documents: Some(deleted_documents),
- edited_documents: Some(edited_documents),
- });
- }
- Err(e) => {
- task.status = Status::Failed;
- task.details = Some(Details::DocumentEdition {
- original_filter,
- context,
- function,
- deleted_documents: Some(0),
- edited_documents: Some(0),
- });
- task.error = Some(e.into());
- }
- }
-
- Ok(vec![task])
- }
- IndexOperation::DocumentDeletion { mut tasks, index_uid: _ } => {
- let mut to_delete = RoaringBitmap::new();
- let external_documents_ids = index.external_documents_ids();
-
- for task in tasks.iter_mut() {
- let before = to_delete.len();
- task.status = Status::Succeeded;
-
- match &task.kind {
- KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
- for id in documents_ids {
- if let Some(id) = external_documents_ids.get(index_wtxn, id)? {
- to_delete.insert(id);
- }
- }
- let will_be_removed = to_delete.len() - before;
- task.details = Some(Details::DocumentDeletion {
- provided_ids: documents_ids.len(),
- deleted_documents: Some(will_be_removed),
- });
- }
- KindWithContent::DocumentDeletionByFilter { index_uid: _, filter_expr } => {
- let before = to_delete.len();
- let filter = match Filter::from_json(filter_expr) {
- Ok(filter) => filter,
- Err(err) => {
- // theorically, this should be catched by deserr before reaching the index-scheduler and cannot happens
- task.status = Status::Failed;
- task.error = match err {
- milli::Error::UserError(
- milli::UserError::InvalidFilterExpression { .. },
- ) => Some(
- Error::from(err)
- .with_custom_error_code(Code::InvalidDocumentFilter)
- .into(),
- ),
- e => Some(e.into()),
- };
- None
- }
- };
- if let Some(filter) = filter {
- let candidates =
- filter.evaluate(index_wtxn, index).map_err(|err| match err {
- milli::Error::UserError(
- milli::UserError::InvalidFilter(_),
- ) => Error::from(err)
- .with_custom_error_code(Code::InvalidDocumentFilter),
- e => e.into(),
- });
- match candidates {
- Ok(candidates) => to_delete |= candidates,
- Err(err) => {
- task.status = Status::Failed;
- task.error = Some(err.into());
- }
- };
- }
- let will_be_removed = to_delete.len() - before;
- if let Some(Details::DocumentDeletionByFilter {
- original_filter: _,
- deleted_documents,
- }) = &mut task.details
- {
- *deleted_documents = Some(will_be_removed);
- } else {
- // In the case of a `documentDeleteByFilter` the details MUST be set
- unreachable!()
- }
- }
- _ => unreachable!(),
- }
- }
-
- if to_delete.is_empty() {
- return Ok(tasks);
- }
-
- let rtxn = index.read_txn()?;
- let db_fields_ids_map = index.fields_ids_map(&rtxn)?;
- let mut new_fields_ids_map = db_fields_ids_map.clone();
-
- // to_delete not empty => index not empty => primary key set
- let primary_key = index.primary_key(&rtxn)?.unwrap();
-
- let primary_key = PrimaryKey::new_or_insert(primary_key, &mut new_fields_ids_map)
- .map_err(milli::Error::from)?;
-
- if !tasks.iter().all(|res| res.error.is_some()) {
- let local_pool;
- let indexer_config = self.index_mapper.indexer_config();
- let pool = match &indexer_config.thread_pool {
- Some(pool) => pool,
- None => {
- local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
- &local_pool
- }
- };
-
- let mut indexer = indexer::DocumentDeletion::new();
- indexer.delete_documents_by_docids(to_delete);
- let document_changes = indexer.into_changes(&indexer_alloc, primary_key);
- let embedders = index.embedding_configs(index_wtxn)?;
- let embedders = self.embedders(embedders)?;
-
- pool.install(|| {
- indexer::index(
- index_wtxn,
- index,
- indexer_config.grenad_parameters(),
- &db_fields_ids_map,
- new_fields_ids_map,
- None, // document deletion never changes primary key
- &document_changes,
- embedders,
- &|| must_stop_processing.get(),
- &send_progress,
- )
- })
- .unwrap()?;
-
- // tracing::info!(indexing_result = ?addition, processed_in = ?started_processing_at.elapsed(), "document indexing done");
- }
-
- Ok(tasks)
- }
- IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
- let indexer_config = self.index_mapper.indexer_config();
- let mut builder = milli::update::Settings::new(index_wtxn, index, indexer_config);
-
- for (task, (_, settings)) in tasks.iter_mut().zip(settings) {
- let checked_settings = settings.clone().check();
- task.details = Some(Details::SettingsUpdate { settings: Box::new(settings) });
- apply_settings_to_builder(&checked_settings, &mut builder);
-
- // We can apply the status right now and if an update fail later
- // the whole batch will be marked as failed.
- task.status = Status::Succeeded;
- }
-
- builder.execute(
- |indexing_step| tracing::debug!(update = ?indexing_step),
- || must_stop_processing.get(),
- )?;
-
- Ok(tasks)
- }
- IndexOperation::SettingsAndDocumentOperation {
- index_uid,
- primary_key,
- method,
- documents_counts,
- operations,
- document_import_tasks,
- settings,
- settings_tasks,
- } => {
- let settings_tasks = self.apply_index_operation(
- index_wtxn,
- index,
- IndexOperation::Settings {
- index_uid: index_uid.clone(),
- settings,
- tasks: settings_tasks,
- },
- )?;
-
- let mut import_tasks = self.apply_index_operation(
- index_wtxn,
- index,
- IndexOperation::DocumentOperation {
- index_uid,
- primary_key,
- method,
- documents_counts,
- operations,
- tasks: document_import_tasks,
- },
- )?;
-
- let mut tasks = settings_tasks;
- tasks.append(&mut import_tasks);
- Ok(tasks)
- }
- IndexOperation::DocumentClearAndSetting {
- index_uid,
- cleared_tasks,
- settings,
- settings_tasks,
- } => {
- let mut import_tasks = self.apply_index_operation(
- index_wtxn,
- index,
- IndexOperation::DocumentClear {
- index_uid: index_uid.clone(),
- tasks: cleared_tasks,
- },
- )?;
-
- let settings_tasks = self.apply_index_operation(
- index_wtxn,
- index,
- IndexOperation::Settings { index_uid, settings, tasks: settings_tasks },
- )?;
-
- let mut tasks = settings_tasks;
- tasks.append(&mut import_tasks);
- Ok(tasks)
- }
- }
- }
-
- /// Delete each given task from all the databases (if it is deleteable).
- ///
- /// Return the number of tasks that were actually deleted.
- fn delete_matched_tasks(
- &self,
- wtxn: &mut RwTxn,
- matched_tasks: &RoaringBitmap,
- ) -> Result {
- // 1. Remove from this list the tasks that we are not allowed to delete
- let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?;
- let processing_tasks = &self.processing_tasks.read().unwrap().processing.clone();
-
- let all_task_ids = self.all_task_ids(wtxn)?;
- let mut to_delete_tasks = all_task_ids & matched_tasks;
- to_delete_tasks -= processing_tasks;
- to_delete_tasks -= enqueued_tasks;
-
- // 2. We now have a list of tasks to delete, delete them
-
- let mut affected_indexes = HashSet::new();
- let mut affected_statuses = HashSet::new();
- let mut affected_kinds = HashSet::new();
- let mut affected_canceled_by = RoaringBitmap::new();
- // The tasks that have been removed *per batches*.
- let mut affected_batches: HashMap = HashMap::new();
-
- for task_id in to_delete_tasks.iter() {
- let task = self.get_task(wtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
-
- affected_indexes.extend(task.indexes().into_iter().map(|x| x.to_owned()));
- affected_statuses.insert(task.status);
- affected_kinds.insert(task.kind.as_kind());
- // Note: don't delete the persisted task data since
- // we can only delete succeeded, failed, and canceled tasks.
- // In each of those cases, the persisted data is supposed to
- // have been deleted already.
- utils::remove_task_datetime(wtxn, self.enqueued_at, task.enqueued_at, task.uid)?;
- if let Some(started_at) = task.started_at {
- utils::remove_task_datetime(wtxn, self.started_at, started_at, task.uid)?;
- }
- if let Some(finished_at) = task.finished_at {
- utils::remove_task_datetime(wtxn, self.finished_at, finished_at, task.uid)?;
- }
- if let Some(canceled_by) = task.canceled_by {
- affected_canceled_by.insert(canceled_by);
- }
- if let Some(batch_uid) = task.batch_uid {
- affected_batches.entry(batch_uid).or_default().insert(task_id);
- }
- }
-
- for index in affected_indexes.iter() {
- self.update_index(wtxn, index, |bitmap| *bitmap -= &to_delete_tasks)?;
- }
-
- for status in affected_statuses.iter() {
- self.update_status(wtxn, *status, |bitmap| *bitmap -= &to_delete_tasks)?;
- }
-
- for kind in affected_kinds.iter() {
- self.update_kind(wtxn, *kind, |bitmap| *bitmap -= &to_delete_tasks)?;
- }
-
- for task in to_delete_tasks.iter() {
- self.all_tasks.delete(wtxn, &task)?;
- }
- for canceled_by in affected_canceled_by {
- if let Some(mut tasks) = self.canceled_by.get(wtxn, &canceled_by)? {
- tasks -= &to_delete_tasks;
- if tasks.is_empty() {
- self.canceled_by.delete(wtxn, &canceled_by)?;
- } else {
- self.canceled_by.put(wtxn, &canceled_by, &tasks)?;
- }
- }
- }
- for (batch_id, to_delete_tasks) in affected_batches {
- if let Some(mut tasks) = self.batch_to_tasks_mapping.get(wtxn, &batch_id)? {
- tasks -= &to_delete_tasks;
- // We must remove the batch entirely
- if tasks.is_empty() {
- self.all_batches.delete(wtxn, &batch_id)?;
- self.batch_to_tasks_mapping.delete(wtxn, &batch_id)?;
- }
- // Anyway, we must remove the batch from all its reverse indexes.
- // The only way to do that is to check
-
- for index in affected_indexes.iter() {
- let index_tasks = self.index_tasks(wtxn, index)?;
- let remaining_index_tasks = index_tasks & &tasks;
- if remaining_index_tasks.is_empty() {
- self.update_batch_index(wtxn, index, |bitmap| {
- bitmap.remove(batch_id);
- })?;
- }
- }
-
- for status in affected_statuses.iter() {
- let status_tasks = self.get_status(wtxn, *status)?;
- let remaining_status_tasks = status_tasks & &tasks;
- if remaining_status_tasks.is_empty() {
- self.update_batch_status(wtxn, *status, |bitmap| {
- bitmap.remove(batch_id);
- })?;
- }
- }
-
- for kind in affected_kinds.iter() {
- let kind_tasks = self.get_kind(wtxn, *kind)?;
- let remaining_kind_tasks = kind_tasks & &tasks;
- if remaining_kind_tasks.is_empty() {
- self.update_batch_kind(wtxn, *kind, |bitmap| {
- bitmap.remove(batch_id);
- })?;
- }
- }
- }
- }
-
- Ok(to_delete_tasks)
- }
-
- /// Cancel each given task from all the databases (if it is cancelable).
- ///
- /// Returns the list of tasks that matched the filter and must be written in the database.
- fn cancel_matched_tasks(
- &self,
- rtxn: &RoTxn,
- cancel_task_id: TaskId,
- current_batch: &mut ProcessingBatch,
- matched_tasks: &RoaringBitmap,
- ) -> Result> {
- // 1. Remove from this list the tasks that we are not allowed to cancel
- // Notice that only the _enqueued_ ones are cancelable and we should
- // have already aborted the indexation of the _processing_ ones
- let cancelable_tasks = self.get_status(rtxn, Status::Enqueued)?;
- let tasks_to_cancel = cancelable_tasks & matched_tasks;
-
- // 2. We now have a list of tasks to cancel, cancel them
- let mut tasks = self.get_existing_tasks(rtxn, tasks_to_cancel.iter())?;
-
- for task in tasks.iter_mut() {
- task.status = Status::Canceled;
- task.canceled_by = Some(cancel_task_id);
- task.details = task.details.as_ref().map(|d| d.to_failed());
- current_batch.processing(Some(task));
- }
-
- Ok(tasks)
- }
-}
diff --git a/crates/index-scheduler/src/dump.rs b/crates/index-scheduler/src/dump.rs
new file mode 100644
index 000000000..ca26e50c8
--- /dev/null
+++ b/crates/index-scheduler/src/dump.rs
@@ -0,0 +1,304 @@
+use std::collections::HashMap;
+use std::io;
+
+use dump::{KindDump, TaskDump, UpdateFile};
+use meilisearch_types::batches::{Batch, BatchId};
+use meilisearch_types::heed::RwTxn;
+use meilisearch_types::milli;
+use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
+use roaring::RoaringBitmap;
+use uuid::Uuid;
+
+use crate::{utils, Error, IndexScheduler, Result};
+
+pub struct Dump<'a> {
+ index_scheduler: &'a IndexScheduler,
+ wtxn: RwTxn<'a>,
+
+ batch_to_task_mapping: HashMap,
+
+ indexes: HashMap,
+ statuses: HashMap,
+ kinds: HashMap,
+
+ batch_indexes: HashMap,
+ batch_statuses: HashMap,
+ batch_kinds: HashMap,
+}
+
+impl<'a> Dump<'a> {
+ pub(crate) fn new(index_scheduler: &'a mut IndexScheduler) -> Result {
+ // While loading a dump no one should be able to access the scheduler thus I can block everything.
+ let wtxn = index_scheduler.env.write_txn()?;
+
+ Ok(Dump {
+ index_scheduler,
+ wtxn,
+ batch_to_task_mapping: HashMap::new(),
+ indexes: HashMap::new(),
+ statuses: HashMap::new(),
+ kinds: HashMap::new(),
+ batch_indexes: HashMap::new(),
+ batch_statuses: HashMap::new(),
+ batch_kinds: HashMap::new(),
+ })
+ }
+
+ /// Register a new batch coming from a dump in the scheduler.
+ /// By taking a mutable ref we're pretty sure no one will ever import a dump while actix is running.
+ pub fn register_dumped_batch(&mut self, batch: Batch) -> Result<()> {
+ self.index_scheduler.queue.batches.all_batches.put(&mut self.wtxn, &batch.uid, &batch)?;
+ if let Some(enqueued_at) = batch.enqueued_at {
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.batches.enqueued_at,
+ enqueued_at.earliest,
+ batch.uid,
+ )?;
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.batches.enqueued_at,
+ enqueued_at.oldest,
+ batch.uid,
+ )?;
+ }
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.batches.started_at,
+ batch.started_at,
+ batch.uid,
+ )?;
+ if let Some(finished_at) = batch.finished_at {
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.batches.finished_at,
+ finished_at,
+ batch.uid,
+ )?;
+ }
+
+ for index in batch.stats.index_uids.keys() {
+ match self.batch_indexes.get_mut(index) {
+ Some(bitmap) => {
+ bitmap.insert(batch.uid);
+ }
+ None => {
+ let mut bitmap = RoaringBitmap::new();
+ bitmap.insert(batch.uid);
+ self.batch_indexes.insert(index.to_string(), bitmap);
+ }
+ };
+ }
+
+ for status in batch.stats.status.keys() {
+ self.batch_statuses.entry(*status).or_default().insert(batch.uid);
+ }
+ for kind in batch.stats.types.keys() {
+ self.batch_kinds.entry(*kind).or_default().insert(batch.uid);
+ }
+
+ Ok(())
+ }
+
+ /// Register a new task coming from a dump in the scheduler.
+ /// By taking a mutable ref we're pretty sure no one will ever import a dump while actix is running.
+ pub fn register_dumped_task(
+ &mut self,
+ task: TaskDump,
+ content_file: Option>,
+ ) -> Result {
+ let task_has_no_docs = matches!(task.kind, KindDump::DocumentImport { documents_count, .. } if documents_count == 0);
+
+ let content_uuid = match content_file {
+ Some(content_file) if task.status == Status::Enqueued => {
+ let (uuid, file) = self.index_scheduler.queue.create_update_file(false)?;
+ let mut writer = io::BufWriter::new(file);
+ for doc in content_file {
+ let doc = doc?;
+ serde_json::to_writer(&mut writer, &doc).map_err(|e| {
+ Error::from_milli(milli::InternalError::SerdeJson(e).into(), None)
+ })?;
+ }
+ let file = writer.into_inner().map_err(|e| e.into_error())?;
+ file.persist()?;
+
+ Some(uuid)
+ }
+ // If the task isn't `Enqueued` then just generate a recognisable `Uuid`
+ // in case we try to open it later.
+ _ if task.status != Status::Enqueued => Some(Uuid::nil()),
+ None if task.status == Status::Enqueued && task_has_no_docs => {
+ let (uuid, file) = self.index_scheduler.queue.create_update_file(false)?;
+ file.persist()?;
+
+ Some(uuid)
+ }
+ _ => None,
+ };
+
+ let task = Task {
+ uid: task.uid,
+ batch_uid: task.batch_uid,
+ enqueued_at: task.enqueued_at,
+ started_at: task.started_at,
+ finished_at: task.finished_at,
+ error: task.error,
+ canceled_by: task.canceled_by,
+ details: task.details,
+ status: task.status,
+ kind: match task.kind {
+ KindDump::DocumentImport {
+ primary_key,
+ method,
+ documents_count,
+ allow_index_creation,
+ } => KindWithContent::DocumentAdditionOrUpdate {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ primary_key,
+ method,
+ content_file: content_uuid.ok_or(Error::CorruptedDump)?,
+ documents_count,
+ allow_index_creation,
+ },
+ KindDump::DocumentDeletion { documents_ids } => KindWithContent::DocumentDeletion {
+ documents_ids,
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ },
+ KindDump::DocumentDeletionByFilter { filter } => {
+ KindWithContent::DocumentDeletionByFilter {
+ filter_expr: filter,
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ }
+ }
+ KindDump::DocumentEdition { filter, context, function } => {
+ KindWithContent::DocumentEdition {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ filter_expr: filter,
+ context,
+ function,
+ }
+ }
+ KindDump::DocumentClear => KindWithContent::DocumentClear {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ },
+ KindDump::Settings { settings, is_deletion, allow_index_creation } => {
+ KindWithContent::SettingsUpdate {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ new_settings: settings,
+ is_deletion,
+ allow_index_creation,
+ }
+ }
+ KindDump::IndexDeletion => KindWithContent::IndexDeletion {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ },
+ KindDump::IndexCreation { primary_key } => KindWithContent::IndexCreation {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ primary_key,
+ },
+ KindDump::IndexUpdate { primary_key } => KindWithContent::IndexUpdate {
+ index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+ primary_key,
+ },
+ KindDump::IndexSwap { swaps } => KindWithContent::IndexSwap { swaps },
+ KindDump::TaskCancelation { query, tasks } => {
+ KindWithContent::TaskCancelation { query, tasks }
+ }
+ KindDump::TasksDeletion { query, tasks } => {
+ KindWithContent::TaskDeletion { query, tasks }
+ }
+ KindDump::DumpCreation { keys, instance_uid } => {
+ KindWithContent::DumpCreation { keys, instance_uid }
+ }
+ KindDump::SnapshotCreation => KindWithContent::SnapshotCreation,
+ KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from },
+ },
+ };
+
+ self.index_scheduler.queue.tasks.all_tasks.put(&mut self.wtxn, &task.uid, &task)?;
+ if let Some(batch_id) = task.batch_uid {
+ self.batch_to_task_mapping.entry(batch_id).or_default().insert(task.uid);
+ }
+
+ for index in task.indexes() {
+ match self.indexes.get_mut(index) {
+ Some(bitmap) => {
+ bitmap.insert(task.uid);
+ }
+ None => {
+ let mut bitmap = RoaringBitmap::new();
+ bitmap.insert(task.uid);
+ self.indexes.insert(index.to_string(), bitmap);
+ }
+ };
+ }
+
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.tasks.enqueued_at,
+ task.enqueued_at,
+ task.uid,
+ )?;
+
+ // we can't override the started_at & finished_at, so we must only set it if the tasks is finished and won't change
+ if matches!(task.status, Status::Succeeded | Status::Failed | Status::Canceled) {
+ if let Some(started_at) = task.started_at {
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.tasks.started_at,
+ started_at,
+ task.uid,
+ )?;
+ }
+ if let Some(finished_at) = task.finished_at {
+ utils::insert_task_datetime(
+ &mut self.wtxn,
+ self.index_scheduler.queue.tasks.finished_at,
+ finished_at,
+ task.uid,
+ )?;
+ }
+ }
+
+ self.statuses.entry(task.status).or_default().insert(task.uid);
+ self.kinds.entry(task.kind.as_kind()).or_default().insert(task.uid);
+
+ Ok(task)
+ }
+
+ /// Commit all the changes and exit the importing dump state
+ pub fn finish(mut self) -> Result<()> {
+ for (batch_id, task_ids) in self.batch_to_task_mapping {
+ self.index_scheduler.queue.batch_to_tasks_mapping.put(
+ &mut self.wtxn,
+ &batch_id,
+ &task_ids,
+ )?;
+ }
+
+ for (index, bitmap) in self.indexes {
+ self.index_scheduler.queue.tasks.index_tasks.put(&mut self.wtxn, &index, &bitmap)?;
+ }
+ for (status, bitmap) in self.statuses {
+ self.index_scheduler.queue.tasks.put_status(&mut self.wtxn, status, &bitmap)?;
+ }
+ for (kind, bitmap) in self.kinds {
+ self.index_scheduler.queue.tasks.put_kind(&mut self.wtxn, kind, &bitmap)?;
+ }
+
+ for (index, bitmap) in self.batch_indexes {
+ self.index_scheduler.queue.batches.index_tasks.put(&mut self.wtxn, &index, &bitmap)?;
+ }
+ for (status, bitmap) in self.batch_statuses {
+ self.index_scheduler.queue.batches.put_status(&mut self.wtxn, status, &bitmap)?;
+ }
+ for (kind, bitmap) in self.batch_kinds {
+ self.index_scheduler.queue.batches.put_kind(&mut self.wtxn, kind, &bitmap)?;
+ }
+
+ self.wtxn.commit()?;
+ self.index_scheduler.scheduler.wake_up.signal();
+
+ Ok(())
+ }
+}
diff --git a/crates/index-scheduler/src/error.rs b/crates/index-scheduler/src/error.rs
index f6a4ecc04..280127d04 100644
--- a/crates/index-scheduler/src/error.rs
+++ b/crates/index-scheduler/src/error.rs
@@ -104,11 +104,13 @@ pub enum Error {
)]
InvalidTaskCanceledBy { canceled_by: String },
#[error(
- "{index_uid} is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 512 bytes."
+ "{index_uid} is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_), and can not be more than 400 bytes."
)]
InvalidIndexUid { index_uid: String },
#[error("Task `{0}` not found.")]
TaskNotFound(TaskId),
+ #[error("Task `{0}` does not contain any documents. Only `documentAdditionOrUpdate` tasks with the statuses `enqueued` or `processing` contain documents")]
+ TaskFileNotFound(TaskId),
#[error("Batch `{0}` not found.")]
BatchNotFound(BatchId),
#[error("Query parameters to filter the tasks to delete are missing. Available query parameters are: `uids`, `indexUids`, `statuses`, `types`, `canceledBy`, `beforeEnqueuedAt`, `afterEnqueuedAt`, `beforeStartedAt`, `afterStartedAt`, `beforeFinishedAt`, `afterFinishedAt`.")]
@@ -122,10 +124,13 @@ pub enum Error {
Dump(#[from] dump::Error),
#[error(transparent)]
Heed(#[from] heed::Error),
- #[error(transparent)]
- Milli(#[from] milli::Error),
- #[error("An unexpected crash occurred when processing the task.")]
- ProcessBatchPanicked,
+ #[error("{}", match .index_uid {
+ Some(uid) if !uid.is_empty() => format!("Index `{}`: {error}", uid),
+ _ => format!("{error}")
+ })]
+ Milli { error: milli::Error, index_uid: Option },
+ #[error("An unexpected crash occurred when processing the task: {0}")]
+ ProcessBatchPanicked(String),
#[error(transparent)]
FileStore(#[from] file_store::Error),
#[error(transparent)]
@@ -144,7 +149,9 @@ pub enum Error {
#[error("Corrupted task queue.")]
CorruptedTaskQueue,
#[error(transparent)]
- TaskDatabaseUpdate(Box),
+ DatabaseUpgrade(Box),
+ #[error(transparent)]
+ UnrecoverableError(Box),
#[error(transparent)]
HeedTransaction(heed::Error),
@@ -184,14 +191,15 @@ impl Error {
| Error::InvalidTaskCanceledBy { .. }
| Error::InvalidIndexUid { .. }
| Error::TaskNotFound(_)
+ | Error::TaskFileNotFound(_)
| Error::BatchNotFound(_)
| Error::TaskDeletionWithEmptyQuery
| Error::TaskCancelationWithEmptyQuery
| Error::AbortedTask
| Error::Dump(_)
| Error::Heed(_)
- | Error::Milli(_)
- | Error::ProcessBatchPanicked
+ | Error::Milli { .. }
+ | Error::ProcessBatchPanicked(_)
| Error::FileStore(_)
| Error::IoError(_)
| Error::Persist(_)
@@ -199,7 +207,8 @@ impl Error {
| Error::Anyhow(_) => true,
Error::CreateBatch(_)
| Error::CorruptedTaskQueue
- | Error::TaskDatabaseUpdate(_)
+ | Error::DatabaseUpgrade(_)
+ | Error::UnrecoverableError(_)
| Error::HeedTransaction(_) => false,
#[cfg(test)]
Error::PlannedFailure => false,
@@ -209,6 +218,20 @@ impl Error {
pub fn with_custom_error_code(self, code: Code) -> Self {
Self::WithCustomErrorCode(code, Box::new(self))
}
+
+ pub fn from_milli(err: milli::Error, index_uid: Option) -> Self {
+ match err {
+ milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
+ Self::Milli { error: err, index_uid }
+ .with_custom_error_code(Code::InvalidDocumentFilter)
+ }
+ milli::Error::UserError(milli::UserError::InvalidFilterExpression { .. }) => {
+ Self::Milli { error: err, index_uid }
+ .with_custom_error_code(Code::InvalidDocumentFilter)
+ }
+ _ => Self::Milli { error: err, index_uid },
+ }
+ }
}
impl ErrorCode for Error {
@@ -230,14 +253,15 @@ impl ErrorCode for Error {
Error::InvalidTaskCanceledBy { .. } => Code::InvalidTaskCanceledBy,
Error::InvalidIndexUid { .. } => Code::InvalidIndexUid,
Error::TaskNotFound(_) => Code::TaskNotFound,
+ Error::TaskFileNotFound(_) => Code::TaskFileNotFound,
Error::BatchNotFound(_) => Code::BatchNotFound,
Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
// TODO: not sure of the Code to use
Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
Error::Dump(e) => e.error_code(),
- Error::Milli(e) => e.error_code(),
- Error::ProcessBatchPanicked => Code::Internal,
+ Error::Milli { error, .. } => error.error_code(),
+ Error::ProcessBatchPanicked(_) => Code::Internal,
Error::Heed(e) => e.error_code(),
Error::HeedTransaction(e) => e.error_code(),
Error::FileStore(e) => e.error_code(),
@@ -249,7 +273,8 @@ impl ErrorCode for Error {
Error::Anyhow(_) => Code::Internal,
Error::CorruptedTaskQueue => Code::Internal,
Error::CorruptedDump => Code::Internal,
- Error::TaskDatabaseUpdate(_) => Code::Internal,
+ Error::DatabaseUpgrade(_) => Code::Internal,
+ Error::UnrecoverableError(_) => Code::Internal,
Error::CreateBatch(_) => Code::Internal,
// This one should never be seen by the end user
diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs
index f4ac80511..109e6b867 100644
--- a/crates/index-scheduler/src/features.rs
+++ b/crates/index-scheduler/src/features.rs
@@ -1,18 +1,29 @@
use std::sync::{Arc, RwLock};
-use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
+use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures};
use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RwTxn};
+use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls};
use crate::error::FeatureNotEnabledError;
use crate::Result;
-const EXPERIMENTAL_FEATURES: &str = "experimental-features";
+/// The number of database used by features
+const NUMBER_OF_DATABASES: u32 = 1;
+/// Database const names for the `FeatureData`.
+mod db_name {
+ pub const EXPERIMENTAL_FEATURES: &str = "experimental-features";
+}
+
+mod db_keys {
+ pub const EXPERIMENTAL_FEATURES: &str = "experimental-features";
+ pub const NETWORK: &str = "network";
+}
#[derive(Clone)]
pub(crate) struct FeatureData {
persisted: Database>,
runtime: Arc>,
+ network: Arc>,
}
#[derive(Debug, Clone, Copy)]
@@ -56,19 +67,6 @@ impl RoFeatures {
}
}
- pub fn check_vector(&self, disabled_action: &'static str) -> Result<()> {
- if self.runtime.vector_store {
- Ok(())
- } else {
- Err(FeatureNotEnabledError {
- disabled_action,
- feature: "vector store",
- issue_link: "https://github.com/meilisearch/product/discussions/677",
- }
- .into())
- }
- }
-
pub fn check_edit_documents_by_function(&self, disabled_action: &'static str) -> Result<()> {
if self.runtime.edit_documents_by_function {
Ok(())
@@ -94,17 +92,62 @@ impl RoFeatures {
.into())
}
}
+
+ pub fn check_network(&self, disabled_action: &'static str) -> Result<()> {
+ if self.runtime.network {
+ Ok(())
+ } else {
+ Err(FeatureNotEnabledError {
+ disabled_action,
+ feature: "network",
+ issue_link: "https://github.com/orgs/meilisearch/discussions/805",
+ }
+ .into())
+ }
+ }
+
+ pub fn check_get_task_documents_route(&self) -> Result<()> {
+ if self.runtime.get_task_documents_route {
+ Ok(())
+ } else {
+ Err(FeatureNotEnabledError {
+ disabled_action: "Getting the documents of an enqueued task",
+ feature: "get task documents route",
+ issue_link: "https://github.com/orgs/meilisearch/discussions/808",
+ }
+ .into())
+ }
+ }
+
+ pub fn check_composite_embedders(&self, disabled_action: &'static str) -> Result<()> {
+ if self.runtime.composite_embedders {
+ Ok(())
+ } else {
+ Err(FeatureNotEnabledError {
+ disabled_action,
+ feature: "composite embedders",
+ issue_link: "https://github.com/orgs/meilisearch/discussions/816",
+ }
+ .into())
+ }
+ }
}
impl FeatureData {
- pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result {
- let mut wtxn = env.write_txn()?;
- let runtime_features_db = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
- wtxn.commit()?;
+ pub(crate) const fn nb_db() -> u32 {
+ NUMBER_OF_DATABASES
+ }
+
+ pub fn new(
+ env: &Env,
+ wtxn: &mut RwTxn,
+ instance_features: InstanceTogglableFeatures,
+ ) -> Result {
+ let runtime_features_db =
+ env.create_database(wtxn, Some(db_name::EXPERIMENTAL_FEATURES))?;
- let txn = env.read_txn()?;
let persisted_features: RuntimeTogglableFeatures =
- runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
+ runtime_features_db.get(wtxn, db_keys::EXPERIMENTAL_FEATURES)?.unwrap_or_default();
let InstanceTogglableFeatures { metrics, logs_route, contains_filter } = instance_features;
let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
metrics: metrics || persisted_features.metrics,
@@ -113,7 +156,14 @@ impl FeatureData {
..persisted_features
}));
- Ok(Self { persisted: runtime_features_db, runtime })
+ let network_db = runtime_features_db.remap_data_type::>();
+ let network: Network = network_db.get(wtxn, db_keys::NETWORK)?.unwrap_or_default();
+
+ Ok(Self {
+ persisted: runtime_features_db,
+ runtime,
+ network: Arc::new(RwLock::new(network)),
+ })
}
pub fn put_runtime_features(
@@ -121,7 +171,7 @@ impl FeatureData {
mut wtxn: RwTxn,
features: RuntimeTogglableFeatures,
) -> Result<()> {
- self.persisted.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
+ self.persisted.put(&mut wtxn, db_keys::EXPERIMENTAL_FEATURES, &features)?;
wtxn.commit()?;
// safe to unwrap, the lock will only fail if:
@@ -142,4 +192,21 @@ impl FeatureData {
pub fn features(&self) -> RoFeatures {
RoFeatures::new(self)
}
+
+ pub fn put_network(&self, mut wtxn: RwTxn, new_network: Network) -> Result<()> {
+ self.persisted.remap_data_type::>().put(
+ &mut wtxn,
+ db_keys::NETWORK,
+ &new_network,
+ )?;
+ wtxn.commit()?;
+
+ let mut network = self.network.write().unwrap();
+ *network = new_network;
+ Ok(())
+ }
+
+ pub fn network(&self) -> Network {
+ Network::clone(&*self.network.read().unwrap())
+ }
}
diff --git a/crates/index-scheduler/src/index_mapper/index_map.rs b/crates/index-scheduler/src/index_mapper/index_map.rs
index f8080d23b..acb95f249 100644
--- a/crates/index-scheduler/src/index_mapper/index_map.rs
+++ b/crates/index-scheduler/src/index_mapper/index_map.rs
@@ -1,16 +1,17 @@
use std::collections::BTreeMap;
+use std::env::VarError;
use std::path::Path;
+use std::str::FromStr;
use std::time::Duration;
use meilisearch_types::heed::{EnvClosingEvent, EnvFlags, EnvOpenOptions};
-use meilisearch_types::milli::Index;
+use meilisearch_types::milli::{Index, Result};
use time::OffsetDateTime;
use uuid::Uuid;
use super::IndexStatus::{self, Available, BeingDeleted, Closing, Missing};
+use crate::clamp_to_page_size;
use crate::lru::{InsertionOutcome, LruMap};
-use crate::{clamp_to_page_size, Result};
-
/// Keep an internally consistent view of the open indexes in memory.
///
/// This view is made of an LRU cache that will evict the least frequently used indexes when new indexes are opened.
@@ -103,7 +104,7 @@ impl ReopenableIndex {
return Ok(());
}
map.unavailable.remove(&self.uuid);
- map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?;
+ map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size, false)?;
}
Ok(())
}
@@ -172,11 +173,12 @@ impl IndexMap {
date: Option<(OffsetDateTime, OffsetDateTime)>,
enable_mdb_writemap: bool,
map_size: usize,
+ creation: bool,
) -> Result {
if !matches!(self.get_unavailable(uuid), Missing) {
panic!("Attempt to open an index that was unavailable");
}
- let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?;
+ let index = create_or_open_index(path, date, enable_mdb_writemap, map_size, creation)?;
match self.available.insert(*uuid, index.clone()) {
InsertionOutcome::InsertedNew => (),
InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
@@ -300,18 +302,31 @@ fn create_or_open_index(
date: Option<(OffsetDateTime, OffsetDateTime)>,
enable_mdb_writemap: bool,
map_size: usize,
+ creation: bool,
) -> Result {
- let mut options = EnvOpenOptions::new();
+ let options = EnvOpenOptions::new();
+ let mut options = options.read_txn_without_tls();
options.map_size(clamp_to_page_size(map_size));
- options.max_readers(1024);
+
+ // You can find more details about this experimental
+ // environment variable on the following GitHub discussion:
+ //
+ let max_readers = match std::env::var("MEILI_EXPERIMENTAL_INDEX_MAX_READERS") {
+ Ok(value) => u32::from_str(&value).unwrap(),
+ Err(VarError::NotPresent) => 1024,
+ Err(VarError::NotUnicode(value)) => panic!(
+ "Invalid unicode for the `MEILI_EXPERIMENTAL_INDEX_MAX_READERS` env var: {value:?}"
+ ),
+ };
+ options.max_readers(max_readers);
if enable_mdb_writemap {
unsafe { options.flags(EnvFlags::WRITE_MAP) };
}
if let Some((created, updated)) = date {
- Ok(Index::new_with_creation_dates(options, path, created, updated)?)
+ Ok(Index::new_with_creation_dates(options, path, created, updated, creation)?)
} else {
- Ok(Index::new(options, path)?)
+ Ok(Index::new(options, path, creation)?)
}
}
@@ -319,17 +334,17 @@ fn create_or_open_index(
#[cfg(test)]
mod tests {
- use meilisearch_types::heed::Env;
+ use meilisearch_types::heed::{Env, WithoutTls};
use meilisearch_types::Index;
use uuid::Uuid;
use super::super::IndexMapper;
- use crate::tests::IndexSchedulerHandle;
+ use crate::test_utils::IndexSchedulerHandle;
use crate::utils::clamp_to_page_size;
use crate::IndexScheduler;
impl IndexMapper {
- fn test() -> (Self, Env, IndexSchedulerHandle) {
+ fn test() -> (Self, Env, IndexSchedulerHandle) {
let (index_scheduler, handle) = IndexScheduler::test(true, vec![]);
(index_scheduler.index_mapper, index_scheduler.env, handle)
}
diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs
index 3cccb5a69..c1f6ff472 100644
--- a/crates/index-scheduler/src/index_mapper/mod.rs
+++ b/crates/index-scheduler/src/index_mapper/mod.rs
@@ -4,7 +4,9 @@ use std::time::Duration;
use std::{fs, thread};
use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
+use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn, WithoutTls};
+use meilisearch_types::milli;
+use meilisearch_types::milli::database_stats::DatabaseStats;
use meilisearch_types::milli::update::IndexerConfig;
use meilisearch_types::milli::{FieldDistribution, Index};
use serde::{Deserialize, Serialize};
@@ -15,12 +17,17 @@ use uuid::Uuid;
use self::index_map::IndexMap;
use self::IndexStatus::{Available, BeingDeleted, Closing, Missing};
use crate::uuid_codec::UuidCodec;
-use crate::{Error, Result};
+use crate::{Error, IndexBudget, IndexSchedulerOptions, Result};
mod index_map;
-const INDEX_MAPPING: &str = "index-mapping";
-const INDEX_STATS: &str = "index-stats";
+/// The number of database used by index mapper
+const NUMBER_OF_DATABASES: u32 = 2;
+/// Database const names for the `IndexMapper`.
+mod db_name {
+ pub const INDEX_MAPPING: &str = "index-mapping";
+ pub const INDEX_STATS: &str = "index-stats";
+}
/// Structure managing meilisearch's indexes.
///
@@ -92,19 +99,32 @@ pub enum IndexStatus {
/// The statistics that can be computed from an `Index` object.
#[derive(Serialize, Deserialize, Debug)]
pub struct IndexStats {
- /// Number of documents in the index.
- pub number_of_documents: u64,
+ /// Stats of the documents database.
+ #[serde(default)]
+ pub documents_database_stats: DatabaseStats,
+
+ #[serde(default, skip_serializing)]
+ pub number_of_documents: Option,
+
/// Size taken up by the index' DB, in bytes.
///
/// This includes the size taken by both the used and free pages of the DB, and as the free pages
/// are not returned to the disk after a deletion, this number is typically larger than
/// `used_database_size` that only includes the size of the used pages.
pub database_size: u64,
+ /// Number of embeddings in the index.
+ /// Option: retrocompatible with the stats of the pre-v1.13.0 versions of meilisearch
+ pub number_of_embeddings: Option,
+ /// Number of embedded documents in the index.
+ /// Option: retrocompatible with the stats of the pre-v1.13.0 versions of meilisearch
+ pub number_of_embedded_documents: Option,
/// Size taken by the used pages of the index' DB, in bytes.
///
/// As the DB backend does not return to the disk the pages that are not currently used by the DB,
/// this value is typically smaller than `database_size`.
pub used_database_size: u64,
+ /// The primary key of the index
+ pub primary_key: Option,
/// Association of every field name with the number of times it occurs in the documents.
pub field_distribution: FieldDistribution,
/// Creation date of the index.
@@ -121,11 +141,16 @@ impl IndexStats {
/// # Parameters
///
/// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`.
- pub fn new(index: &Index, rtxn: &RoTxn) -> Result {
+ pub fn new(index: &Index, rtxn: &RoTxn) -> milli::Result {
+ let arroy_stats = index.arroy_stats(rtxn)?;
Ok(IndexStats {
- number_of_documents: index.number_of_documents(rtxn)?,
+ number_of_embeddings: Some(arroy_stats.number_of_embeddings),
+ number_of_embedded_documents: Some(arroy_stats.documents.len()),
+ documents_database_stats: index.documents_stats(rtxn)?.unwrap_or_default(),
+ number_of_documents: None,
database_size: index.on_disk_size()?,
used_database_size: index.used_size()?,
+ primary_key: index.primary_key(rtxn)?.map(|s| s.to_string()),
field_distribution: index.field_distribution(rtxn)?,
created_at: index.created_at(rtxn)?,
updated_at: index.updated_at(rtxn)?,
@@ -134,29 +159,25 @@ impl IndexStats {
}
impl IndexMapper {
- pub fn new(
- env: &Env,
- base_path: PathBuf,
- index_base_map_size: usize,
- index_growth_amount: usize,
- index_count: usize,
- enable_mdb_writemap: bool,
- indexer_config: IndexerConfig,
- ) -> Result {
- let mut wtxn = env.write_txn()?;
- let index_mapping = env.create_database(&mut wtxn, Some(INDEX_MAPPING))?;
- let index_stats = env.create_database(&mut wtxn, Some(INDEX_STATS))?;
- wtxn.commit()?;
+ pub(crate) const fn nb_db() -> u32 {
+ NUMBER_OF_DATABASES
+ }
+ pub fn new(
+ env: &Env,
+ wtxn: &mut RwTxn,
+ options: &IndexSchedulerOptions,
+ budget: IndexBudget,
+ ) -> Result {
Ok(Self {
- index_map: Arc::new(RwLock::new(IndexMap::new(index_count))),
- index_mapping,
- index_stats,
- base_path,
- index_base_map_size,
- index_growth_amount,
- enable_mdb_writemap,
- indexer_config: Arc::new(indexer_config),
+ index_map: Arc::new(RwLock::new(IndexMap::new(budget.index_count))),
+ index_mapping: env.create_database(wtxn, Some(db_name::INDEX_MAPPING))?,
+ index_stats: env.create_database(wtxn, Some(db_name::INDEX_STATS))?,
+ base_path: options.indexes_path.clone(),
+ index_base_map_size: budget.map_size,
+ index_growth_amount: options.index_growth_amount,
+ enable_mdb_writemap: options.enable_mdb_writemap,
+ indexer_config: options.indexer_config.clone(),
currently_updating_index: Default::default(),
})
}
@@ -183,13 +204,24 @@ impl IndexMapper {
// Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
// This is very unlikely to happen in practice.
// TODO: it would be better to lazily create the index. But we need an Index::open function for milli.
- let index = self.index_map.write().unwrap().create(
- &uuid,
- &index_path,
- date,
- self.enable_mdb_writemap,
- self.index_base_map_size,
- )?;
+ let index = self
+ .index_map
+ .write()
+ .unwrap()
+ .create(
+ &uuid,
+ &index_path,
+ date,
+ self.enable_mdb_writemap,
+ self.index_base_map_size,
+ true,
+ )
+ .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
+ let index_rtxn = index.read_txn()?;
+ let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)
+ .map_err(|e| Error::from_milli(e, Some(name.to_string())))?;
+ self.store_stats_of(&mut wtxn, name, &stats)?;
+ drop(index_rtxn);
wtxn.commit()?;
@@ -357,7 +389,9 @@ impl IndexMapper {
};
let index_path = self.base_path.join(uuid.to_string());
// take the lock to reopen the environment.
- reopen.reopen(&mut self.index_map.write().unwrap(), &index_path)?;
+ reopen
+ .reopen(&mut self.index_map.write().unwrap(), &index_path)
+ .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
continue;
}
BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
@@ -372,13 +406,16 @@ impl IndexMapper {
Missing => {
let index_path = self.base_path.join(uuid.to_string());
- break index_map.create(
- &uuid,
- &index_path,
- None,
- self.enable_mdb_writemap,
- self.index_base_map_size,
- )?;
+ break index_map
+ .create(
+ &uuid,
+ &index_path,
+ None,
+ self.enable_mdb_writemap,
+ self.index_base_map_size,
+ false,
+ )
+ .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
}
Available(index) => break index,
Closing(_) => {
@@ -460,6 +497,7 @@ impl IndexMapper {
let index = self.index(rtxn, index_uid)?;
let index_rtxn = index.read_txn()?;
IndexStats::new(&index, &index_rtxn)
+ .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))
}
}
}
diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs
index bcd5966b5..bcc295afd 100644
--- a/crates/index-scheduler/src/insta_snapshot.rs
+++ b/crates/index-scheduler/src/insta_snapshot.rs
@@ -1,15 +1,16 @@
use std::collections::BTreeSet;
use std::fmt::Write;
-use meilisearch_types::batches::Batch;
+use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats};
use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str};
use meilisearch_types::heed::{Database, RoTxn};
use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
-use meilisearch_types::tasks::{Details, Task};
+use meilisearch_types::tasks::{Details, Kind, Status, Task};
+use meilisearch_types::versioning;
use roaring::RoaringBitmap;
use crate::index_mapper::IndexMapper;
-use crate::{IndexScheduler, Kind, Status, BEI128};
+use crate::{IndexScheduler, BEI128};
pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
// Since we'll snapshot the index right afterward, we don't need to ensure it's internally consistent for every run.
@@ -18,41 +19,15 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
scheduler.assert_internally_consistent();
let IndexScheduler {
- autobatching_enabled,
cleanup_enabled: _,
- must_stop_processing: _,
processing_tasks,
- file_store,
env,
- all_tasks,
- all_batches,
- batch_to_tasks_mapping,
- // task reverse index
- status,
- kind,
- index_tasks,
- canceled_by,
- enqueued_at,
- started_at,
- finished_at,
-
- // batch reverse index
- batch_status,
- batch_kind,
- batch_index_tasks,
- batch_enqueued_at,
- batch_started_at,
- batch_finished_at,
+ version,
+ queue,
+ scheduler,
index_mapper,
features: _,
- max_number_of_tasks: _,
- max_number_of_batched_tasks: _,
- wake_up: _,
- dumps_path: _,
- snapshots_path: _,
- auth_path: _,
- version_file_path: _,
webhook_url: _,
webhook_authorization_header: _,
test_breakpoint_sdr: _,
@@ -65,8 +40,18 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
let mut snap = String::new();
+ let indx_sched_version = version.get_version(&rtxn).unwrap();
+ let latest_version = (
+ versioning::VERSION_MAJOR.parse().unwrap(),
+ versioning::VERSION_MINOR.parse().unwrap(),
+ versioning::VERSION_PATCH.parse().unwrap(),
+ );
+ if indx_sched_version != Some(latest_version) {
+ snap.push_str(&format!("index scheduler running on version {indx_sched_version:?}\n"));
+ }
+
let processing = processing_tasks.read().unwrap().clone();
- snap.push_str(&format!("### Autobatching Enabled = {autobatching_enabled}\n"));
+ snap.push_str(&format!("### Autobatching Enabled = {}\n", scheduler.autobatching_enabled));
snap.push_str(&format!(
"### Processing batch {:?}:\n",
processing.batch.as_ref().map(|batch| batch.uid)
@@ -79,19 +64,19 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
snap.push_str("\n----------------------------------------------------------------------\n");
snap.push_str("### All Tasks:\n");
- snap.push_str(&snapshot_all_tasks(&rtxn, *all_tasks));
+ snap.push_str(&snapshot_all_tasks(&rtxn, queue.tasks.all_tasks));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Status:\n");
- snap.push_str(&snapshot_status(&rtxn, *status));
+ snap.push_str(&snapshot_status(&rtxn, queue.tasks.status));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Kind:\n");
- snap.push_str(&snapshot_kind(&rtxn, *kind));
+ snap.push_str(&snapshot_kind(&rtxn, queue.tasks.kind));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Index Tasks:\n");
- snap.push_str(&snapshot_index_tasks(&rtxn, *index_tasks));
+ snap.push_str(&snapshot_index_tasks(&rtxn, queue.tasks.index_tasks));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Index Mapper:\n");
@@ -99,55 +84,55 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
snap.push_str("\n----------------------------------------------------------------------\n");
snap.push_str("### Canceled By:\n");
- snap.push_str(&snapshot_canceled_by(&rtxn, *canceled_by));
+ snap.push_str(&snapshot_canceled_by(&rtxn, queue.tasks.canceled_by));
snap.push_str("\n----------------------------------------------------------------------\n");
snap.push_str("### Enqueued At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *enqueued_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.tasks.enqueued_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Started At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *started_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.tasks.started_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Finished At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *finished_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.tasks.finished_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### All Batches:\n");
- snap.push_str(&snapshot_all_batches(&rtxn, *all_batches));
+ snap.push_str(&snapshot_all_batches(&rtxn, queue.batches.all_batches));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batch to tasks mapping:\n");
- snap.push_str(&snapshot_batches_to_tasks_mappings(&rtxn, *batch_to_tasks_mapping));
+ snap.push_str(&snapshot_batches_to_tasks_mappings(&rtxn, queue.batch_to_tasks_mapping));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Status:\n");
- snap.push_str(&snapshot_status(&rtxn, *batch_status));
+ snap.push_str(&snapshot_status(&rtxn, queue.batches.status));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Kind:\n");
- snap.push_str(&snapshot_kind(&rtxn, *batch_kind));
+ snap.push_str(&snapshot_kind(&rtxn, queue.batches.kind));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Index Tasks:\n");
- snap.push_str(&snapshot_index_tasks(&rtxn, *batch_index_tasks));
+ snap.push_str(&snapshot_index_tasks(&rtxn, queue.batches.index_tasks));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Enqueued At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *batch_enqueued_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.batches.enqueued_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Started At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *batch_started_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.batches.started_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### Batches Finished At:\n");
- snap.push_str(&snapshot_date_db(&rtxn, *batch_finished_at));
+ snap.push_str(&snapshot_date_db(&rtxn, queue.batches.finished_at));
snap.push_str("----------------------------------------------------------------------\n");
snap.push_str("### File Store:\n");
- snap.push_str(&snapshot_file_store(file_store));
+ snap.push_str(&snapshot_file_store(&queue.file_store));
snap.push_str("\n----------------------------------------------------------------------\n");
snap
@@ -306,6 +291,9 @@ fn snapshot_details(d: &Details) -> String {
Details::IndexSwap { swaps } => {
format!("{{ swaps: {swaps:?} }}")
}
+ Details::UpgradeDatabase { from, to } => {
+ format!("{{ from: {from:?}, to: {to:?} }}")
+ }
}
}
@@ -353,14 +341,23 @@ pub fn snapshot_canceled_by(rtxn: &RoTxn, db: Database String {
let mut snap = String::new();
- let Batch { uid, details, stats, started_at, finished_at } = batch;
+ let Batch { uid, details, stats, started_at, finished_at, progress: _, enqueued_at } = batch;
+ let stats = BatchStats {
+ progress_trace: Default::default(),
+ write_channel_congestion: None,
+ ..stats.clone()
+ };
if let Some(finished_at) = finished_at {
assert!(finished_at > started_at);
}
+ let BatchEnqueuedAt { earliest, oldest } = enqueued_at.unwrap();
+ assert!(*started_at > earliest);
+ assert!(earliest >= oldest);
+
snap.push('{');
snap.push_str(&format!("uid: {uid}, "));
snap.push_str(&format!("details: {}, ", serde_json::to_string(details).unwrap()));
- snap.push_str(&format!("stats: {}, ", serde_json::to_string(stats).unwrap()));
+ snap.push_str(&format!("stats: {}, ", serde_json::to_string(&stats).unwrap()));
snap.push('}');
snap
}
@@ -373,7 +370,8 @@ pub fn snapshot_index_mapper(rtxn: &RoTxn, mapper: &IndexMapper) -> String {
let stats = mapper.stats_of(rtxn, &name).unwrap();
s.push_str(&format!(
"{name}: {{ number_of_documents: {}, field_distribution: {:?} }}\n",
- stats.number_of_documents, stats.field_distribution
+ stats.documents_database_stats.number_of_entries(),
+ stats.field_distribution
));
}
diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs
index cef24c1ea..feb08316c 100644
--- a/crates/index-scheduler/src/lib.rs
+++ b/crates/index-scheduler/src/lib.rs
@@ -1,7 +1,7 @@
/*!
This crate defines the index scheduler, which is responsible for:
1. Keeping references to meilisearch's indexes and mapping them to their
-user-defined names.
+ user-defined names.
2. Scheduling tasks given by the user and executing them, in batch if possible.
When an `IndexScheduler` is created, a new thread containing a reference to the
@@ -18,247 +18,62 @@ called asynchronously from any thread. These methods can either query the
content of the scheduler or enqueue new tasks.
*/
-mod autobatcher;
-mod batch;
+mod dump;
pub mod error;
mod features;
mod index_mapper;
#[cfg(test)]
mod insta_snapshot;
mod lru;
+mod processing;
+mod queue;
+mod scheduler;
+#[cfg(test)]
+mod test_utils;
+pub mod upgrade;
mod utils;
pub mod uuid_codec;
+pub mod versioning;
-pub type Result = std::result::Result;
+pub type Result = std::result::Result;
pub type TaskId = u32;
use std::collections::{BTreeMap, HashMap};
use std::io::{self, BufReader, Read};
-use std::ops::{Bound, RangeBounds};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::path::{Path, PathBuf};
-use std::sync::atomic::Ordering::{self, Relaxed};
-use std::sync::atomic::{AtomicBool, AtomicU32};
use std::sync::{Arc, RwLock};
use std::time::Duration;
-use dump::{KindDump, TaskDump, UpdateFile};
+use dump::Dump;
pub use error::Error;
pub use features::RoFeatures;
-use file_store::FileStore;
use flate2::bufread::GzEncoder;
use flate2::Compression;
-use meilisearch_types::batches::{Batch, BatchId};
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
+use meilisearch_types::batches::Batch;
+use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures};
use meilisearch_types::heed::byteorder::BE;
-use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str, I128};
-use meilisearch_types::heed::{self, Database, Env, PutFlags, RoTxn, RwTxn};
-use meilisearch_types::milli::documents::DocumentsBatchBuilder;
+use meilisearch_types::heed::types::I128;
+use meilisearch_types::heed::{self, Env, RoTxn, WithoutTls};
use meilisearch_types::milli::index::IndexEmbeddingConfig;
-use meilisearch_types::milli::update::new::indexer::document_changes::Progress;
use meilisearch_types::milli::update::IndexerConfig;
use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs};
-use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
+use meilisearch_types::milli::{self, Index};
use meilisearch_types::task_view::TaskView;
-use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task, TaskProgress};
-use rayon::current_num_threads;
-use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+use meilisearch_types::tasks::{KindWithContent, Task};
+use processing::ProcessingTasks;
+pub use queue::Query;
+use queue::Queue;
use roaring::RoaringBitmap;
-use synchronoise::SignalEvent;
-use time::format_description::well_known::Rfc3339;
+use scheduler::Scheduler;
use time::OffsetDateTime;
-use utils::{filter_out_references_to_newer_tasks, keep_ids_within_datetimes, map_bound};
-use uuid::Uuid;
+use versioning::Versioning;
use crate::index_mapper::IndexMapper;
-use crate::utils::{check_index_swap_validity, clamp_to_page_size, ProcessingBatch};
+use crate::utils::clamp_to_page_size;
pub(crate) type BEI128 = I128;
-/// Defines a subset of tasks to be retrieved from the [`IndexScheduler`].
-///
-/// An empty/default query (where each field is set to `None`) matches all tasks.
-/// Each non-null field restricts the set of tasks further.
-#[derive(Default, Debug, Clone, PartialEq, Eq)]
-pub struct Query {
- /// The maximum number of tasks to be matched
- pub limit: Option,
- /// The minimum [task id](`meilisearch_types::tasks::Task::uid`) to be matched
- pub from: Option,
- /// The order used to return the tasks. By default the newest tasks are returned first and the boolean is `false`.
- pub reverse: Option,
- /// The [task ids](`meilisearch_types::tasks::Task::uid`) to be matched
- pub uids: Option>,
- /// The [batch ids](`meilisearch_types::batches::Batch::uid`) to be matched
- pub batch_uids: Option>,
- /// The allowed [statuses](`meilisearch_types::tasks::Task::status`) of the matched tasls
- pub statuses: Option>,
- /// The allowed [kinds](meilisearch_types::tasks::Kind) of the matched tasks.
- ///
- /// The kind of a task is given by:
- /// ```
- /// # use meilisearch_types::tasks::{Task, Kind};
- /// # fn doc_func(task: Task) -> Kind {
- /// task.kind.as_kind()
- /// # }
- /// ```
- pub types: Option>,
- /// The allowed [index ids](meilisearch_types::tasks::Task::index_uid) of the matched tasks
- pub index_uids: Option>,
- /// The [task ids](`meilisearch_types::tasks::Task::uid`) of the [`TaskCancelation`](meilisearch_types::tasks::Task::Kind::TaskCancelation) tasks
- /// that canceled the matched tasks.
- pub canceled_by: Option>,
- /// Exclusive upper bound of the matched tasks' [`enqueued_at`](meilisearch_types::tasks::Task::enqueued_at) field.
- pub before_enqueued_at: Option,
- /// Exclusive lower bound of the matched tasks' [`enqueued_at`](meilisearch_types::tasks::Task::enqueued_at) field.
- pub after_enqueued_at: Option,
- /// Exclusive upper bound of the matched tasks' [`started_at`](meilisearch_types::tasks::Task::started_at) field.
- pub before_started_at: Option,
- /// Exclusive lower bound of the matched tasks' [`started_at`](meilisearch_types::tasks::Task::started_at) field.
- pub after_started_at: Option,
- /// Exclusive upper bound of the matched tasks' [`finished_at`](meilisearch_types::tasks::Task::finished_at) field.
- pub before_finished_at: Option,
- /// Exclusive lower bound of the matched tasks' [`finished_at`](meilisearch_types::tasks::Task::finished_at) field.
- pub after_finished_at: Option,
-}
-
-impl Query {
- /// Return `true` if every field of the query is set to `None`, such that the query
- /// matches all tasks.
- pub fn is_empty(&self) -> bool {
- matches!(
- self,
- Query {
- limit: None,
- from: None,
- reverse: None,
- uids: None,
- batch_uids: None,
- statuses: None,
- types: None,
- index_uids: None,
- canceled_by: None,
- before_enqueued_at: None,
- after_enqueued_at: None,
- before_started_at: None,
- after_started_at: None,
- before_finished_at: None,
- after_finished_at: None,
- }
- )
- }
-
- /// Add an [index id](meilisearch_types::tasks::Task::index_uid) to the list of permitted indexes.
- pub fn with_index(self, index_uid: String) -> Self {
- let mut index_vec = self.index_uids.unwrap_or_default();
- index_vec.push(index_uid);
- Self { index_uids: Some(index_vec), ..self }
- }
-
- // Removes the `from` and `limit` restrictions from the query.
- // Useful to get the total number of tasks matching a filter.
- pub fn without_limits(self) -> Self {
- Query { limit: None, from: None, ..self }
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct ProcessingTasks {
- batch: Option,
- /// The list of tasks ids that are currently running.
- processing: RoaringBitmap,
- /// The progress on processing tasks
- progress: Option,
-}
-
-impl ProcessingTasks {
- /// Creates an empty `ProcessingAt` struct.
- fn new() -> ProcessingTasks {
- ProcessingTasks { batch: None, processing: RoaringBitmap::new(), progress: None }
- }
-
- /// Stores the currently processing tasks, and the date time at which it started.
- fn start_processing(&mut self, processing_batch: ProcessingBatch, processing: RoaringBitmap) {
- self.batch = Some(processing_batch);
- self.processing = processing;
- }
-
- fn update_progress(&mut self, progress: Progress) -> TaskProgress {
- self.progress.get_or_insert_with(TaskProgress::default).update(progress)
- }
-
- /// Set the processing tasks to an empty list
- fn stop_processing(&mut self) -> Self {
- self.progress = None;
-
- Self {
- batch: std::mem::take(&mut self.batch),
- processing: std::mem::take(&mut self.processing),
- progress: None,
- }
- }
-
- /// Returns `true` if there, at least, is one task that is currently processing that we must stop.
- fn must_cancel_processing_tasks(&self, canceled_tasks: &RoaringBitmap) -> bool {
- !self.processing.is_disjoint(canceled_tasks)
- }
-}
-
-#[derive(Default, Clone, Debug)]
-struct MustStopProcessing(Arc);
-
-impl MustStopProcessing {
- fn get(&self) -> bool {
- self.0.load(Relaxed)
- }
-
- fn must_stop(&self) {
- self.0.store(true, Relaxed);
- }
-
- fn reset(&self) {
- self.0.store(false, Relaxed);
- }
-}
-
-/// Database const names for the `IndexScheduler`.
-mod db_name {
- pub const ALL_TASKS: &str = "all-tasks";
- pub const ALL_BATCHES: &str = "all-batches";
- pub const BATCH_TO_TASKS_MAPPING: &str = "batch-to-tasks-mapping";
- pub const STATUS: &str = "status";
- pub const KIND: &str = "kind";
- pub const INDEX_TASKS: &str = "index-tasks";
- pub const CANCELED_BY: &str = "canceled_by";
- pub const ENQUEUED_AT: &str = "enqueued-at";
- pub const STARTED_AT: &str = "started-at";
- pub const FINISHED_AT: &str = "finished-at";
-
- pub const BATCH_STATUS: &str = "batch-status";
- pub const BATCH_KIND: &str = "batch-kind";
- pub const BATCH_INDEX_TASKS: &str = "batch-index-tasks";
- pub const BATCH_ENQUEUED_AT: &str = "batch-enqueued-at";
- pub const BATCH_STARTED_AT: &str = "batch-started-at";
- pub const BATCH_FINISHED_AT: &str = "batch-finished-at";
-}
-
-#[cfg(test)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Breakpoint {
- // this state is only encountered while creating the scheduler in the test suite.
- Init,
-
- Start,
- BatchCreated,
- BeforeProcessing,
- AfterProcessing,
- AbortedIndexation,
- ProcessBatchSucceeded,
- ProcessBatchFailed,
- InsideProcessBatch,
-}
-
#[derive(Debug)]
pub struct IndexSchedulerOptions {
/// The path to the version file of Meilisearch.
@@ -290,7 +105,7 @@ pub struct IndexSchedulerOptions {
/// The number of indexes that can be concurrently opened in memory.
pub index_count: usize,
/// Configuration used during indexing for each meilisearch index.
- pub indexer_config: IndexerConfig,
+ pub indexer_config: Arc,
/// Set to `true` iff the index scheduler is allowed to automatically
/// batch tasks together, to process multiple tasks at once.
pub autobatching_enabled: bool,
@@ -303,102 +118,53 @@ pub struct IndexSchedulerOptions {
/// If the autobatcher is allowed to automatically batch tasks
/// it will only batch this defined number of tasks at once.
pub max_number_of_batched_tasks: usize,
+ /// If the autobatcher is allowed to automatically batch tasks
+ /// it will only batch this defined maximum size (in bytes) of tasks at once.
+ pub batched_tasks_size_limit: u64,
/// The experimental features enabled for this instance.
pub instance_features: InstanceTogglableFeatures,
+ /// The experimental features enabled for this instance.
+ pub auto_upgrade: bool,
+ /// The maximal number of entries in the search query cache of an embedder.
+ ///
+ /// 0 disables the cache.
+ pub embedding_cache_cap: usize,
}
/// Structure which holds meilisearch's indexes and schedules the tasks
/// to be performed on them.
pub struct IndexScheduler {
/// The LMDB environment which the DBs are associated with.
- pub(crate) env: Env,
-
- /// A boolean that can be set to true to stop the currently processing tasks.
- pub(crate) must_stop_processing: MustStopProcessing,
+ pub(crate) env: Env,
/// The list of tasks currently processing
pub(crate) processing_tasks: Arc>,
- /// The list of files referenced by the tasks
- pub(crate) file_store: FileStore,
-
- /// The main database, it contains all the tasks accessible by their Id.
- pub(crate) all_tasks: Database>,
-
- /// Contains all the batches accessible by their Id.
- pub(crate) all_batches: Database>,
-
- /// Matches a batch id with the associated task ids.
- pub(crate) batch_to_tasks_mapping: Database,
-
- /// All the tasks ids grouped by their status.
- // TODO we should not be able to serialize a `Status::Processing` in this database.
- pub(crate) status: Database, RoaringBitmapCodec>,
- /// All the tasks ids grouped by their kind.
- pub(crate) kind: Database, RoaringBitmapCodec>,
- /// Store the tasks associated to an index.
- pub(crate) index_tasks: Database,
- /// Store the tasks that were canceled by a task uid
- pub(crate) canceled_by: Database,
- /// Store the task ids of tasks which were enqueued at a specific date
- pub(crate) enqueued_at: Database,
- /// Store the task ids of finished tasks which started being processed at a specific date
- pub(crate) started_at: Database,
- /// Store the task ids of tasks which finished at a specific date
- pub(crate) finished_at: Database,
-
- /// All the batches containing a task matching the selected status.
- pub(crate) batch_status: Database, RoaringBitmapCodec>,
- /// All the batches ids grouped by the kind of their task.
- pub(crate) batch_kind: Database, RoaringBitmapCodec>,
- /// Store the batches associated to an index.
- pub(crate) batch_index_tasks: Database,
- /// Store the batches containing tasks which were enqueued at a specific date
- pub(crate) batch_enqueued_at: Database,
- /// Store the batches containing finished tasks started at a specific date
- pub(crate) batch_started_at: Database,
- /// Store the batches containing tasks finished at a specific date
- pub(crate) batch_finished_at: Database,
-
+ /// A database containing only the version of the index-scheduler
+ pub version: versioning::Versioning,
+ /// The queue containing both the tasks and the batches.
+ pub queue: queue::Queue,
/// In charge of creating, opening, storing and returning indexes.
pub(crate) index_mapper: IndexMapper,
-
/// In charge of fetching and setting the status of experimental features.
features: features::FeatureData,
- /// Get a signal when a batch needs to be processed.
- pub(crate) wake_up: Arc,
-
- /// Whether auto-batching is enabled or not.
- pub(crate) autobatching_enabled: bool,
+ /// Everything related to the processing of the tasks
+ pub scheduler: scheduler::Scheduler,
/// Whether we should automatically cleanup the task queue or not.
pub(crate) cleanup_enabled: bool,
- /// The max number of tasks allowed before the scheduler starts to delete
- /// the finished tasks automatically.
- pub(crate) max_number_of_tasks: usize,
-
- /// The maximum number of tasks that will be batched together.
- pub(crate) max_number_of_batched_tasks: usize,
-
/// The webhook url we should send tasks to after processing every batches.
pub(crate) webhook_url: Option,
/// The Authorization header to send to the webhook URL.
pub(crate) webhook_authorization_header: Option,
- /// The path used to create the dumps.
- pub(crate) dumps_path: PathBuf,
-
- /// The path used to create the snapshots.
- pub(crate) snapshots_path: PathBuf,
-
- /// The path to the folder containing the auth LMDB env.
- pub(crate) auth_path: PathBuf,
-
- /// The path to the version file of Meilisearch.
- pub(crate) version_file_path: PathBuf,
-
+ /// A map to retrieve the runtime representation of an embedder depending on its configuration.
+ ///
+ /// This map may return the same embedder object for two different indexes or embedder settings,
+ /// but it will only do this if the embedder configuration options are the same, leading
+ /// to the same embeddings for the same input text.
embedders: Arc>>>,
// ================= test
@@ -407,13 +173,13 @@ pub struct IndexScheduler {
///
/// See [self.breakpoint()](`IndexScheduler::breakpoint`) for an explanation.
#[cfg(test)]
- test_breakpoint_sdr: crossbeam::channel::Sender<(Breakpoint, bool)>,
+ test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
/// A list of planned failures within the [`tick`](IndexScheduler::tick) method of the index scheduler.
///
/// The first field is the iteration index and the second field identifies a location in the code.
#[cfg(test)]
- planned_failures: Vec<(usize, tests::FailureLocation)>,
+ planned_failures: Vec<(usize, test_utils::FailureLocation)>,
/// A counter that is incremented before every call to [`tick`](IndexScheduler::tick)
#[cfg(test)]
@@ -424,40 +190,13 @@ impl IndexScheduler {
fn private_clone(&self) -> IndexScheduler {
IndexScheduler {
env: self.env.clone(),
- must_stop_processing: self.must_stop_processing.clone(),
processing_tasks: self.processing_tasks.clone(),
- file_store: self.file_store.clone(),
- all_tasks: self.all_tasks,
- all_batches: self.all_batches,
- batch_to_tasks_mapping: self.batch_to_tasks_mapping,
-
- // Tasks reverse index
- status: self.status,
- kind: self.kind,
- index_tasks: self.index_tasks,
- canceled_by: self.canceled_by,
- enqueued_at: self.enqueued_at,
- started_at: self.started_at,
- finished_at: self.finished_at,
-
- // Batches reverse index
- batch_status: self.batch_status,
- batch_kind: self.batch_kind,
- batch_index_tasks: self.batch_index_tasks,
- batch_enqueued_at: self.batch_enqueued_at,
- batch_started_at: self.batch_started_at,
- batch_finished_at: self.batch_finished_at,
+ version: self.version.clone(),
+ queue: self.queue.private_clone(),
+ scheduler: self.scheduler.private_clone(),
index_mapper: self.index_mapper.clone(),
- wake_up: self.wake_up.clone(),
- autobatching_enabled: self.autobatching_enabled,
cleanup_enabled: self.cleanup_enabled,
- max_number_of_tasks: self.max_number_of_tasks,
- max_number_of_batched_tasks: self.max_number_of_batched_tasks,
- snapshots_path: self.snapshots_path.clone(),
- dumps_path: self.dumps_path.clone(),
- auth_path: self.auth_path.clone(),
- version_file_path: self.version_file_path.clone(),
webhook_url: self.webhook_url.clone(),
webhook_authorization_header: self.webhook_authorization_header.clone(),
embedders: self.embedders.clone(),
@@ -470,14 +209,19 @@ impl IndexScheduler {
features: self.features.clone(),
}
}
-}
-impl IndexScheduler {
+ pub(crate) const fn nb_db() -> u32 {
+ Versioning::nb_db() + Queue::nb_db() + IndexMapper::nb_db() + features::FeatureData::nb_db()
+ }
+
/// Create an index scheduler and start its run loop.
+ #[allow(private_interfaces)] // because test_utils is private
pub fn new(
options: IndexSchedulerOptions,
- #[cfg(test)] test_breakpoint_sdr: crossbeam::channel::Sender<(Breakpoint, bool)>,
- #[cfg(test)] planned_failures: Vec<(usize, tests::FailureLocation)>,
+ auth_env: Env,
+ from_db_version: (u32, u32, u32),
+ #[cfg(test)] test_breakpoint_sdr: crossbeam_channel::Sender<(test_utils::Breakpoint, bool)>,
+ #[cfg(test)] planned_failures: Vec<(usize, test_utils::FailureLocation)>,
) -> Result {
std::fs::create_dir_all(&options.tasks_path)?;
std::fs::create_dir_all(&options.update_file_path)?;
@@ -506,83 +250,33 @@ impl IndexScheduler {
};
let env = unsafe {
- heed::EnvOpenOptions::new()
- .max_dbs(19)
+ let env_options = heed::EnvOpenOptions::new();
+ let mut env_options = env_options.read_txn_without_tls();
+ env_options
+ .max_dbs(Self::nb_db())
.map_size(budget.task_db_size)
- .open(options.tasks_path)
+ .open(&options.tasks_path)
}?;
- let features = features::FeatureData::new(&env, options.instance_features)?;
-
- let file_store = FileStore::new(&options.update_file_path)?;
+ // We **must** starts by upgrading the version because it'll also upgrade the required database before we can open them
+ let version = versioning::Versioning::new(&env, from_db_version)?;
let mut wtxn = env.write_txn()?;
- let all_tasks = env.create_database(&mut wtxn, Some(db_name::ALL_TASKS))?;
- let all_batches = env.create_database(&mut wtxn, Some(db_name::ALL_BATCHES))?;
- let batch_to_tasks_mapping =
- env.create_database(&mut wtxn, Some(db_name::BATCH_TO_TASKS_MAPPING))?;
-
- let status = env.create_database(&mut wtxn, Some(db_name::STATUS))?;
- let kind = env.create_database(&mut wtxn, Some(db_name::KIND))?;
- let index_tasks = env.create_database(&mut wtxn, Some(db_name::INDEX_TASKS))?;
- let canceled_by = env.create_database(&mut wtxn, Some(db_name::CANCELED_BY))?;
- let enqueued_at = env.create_database(&mut wtxn, Some(db_name::ENQUEUED_AT))?;
- let started_at = env.create_database(&mut wtxn, Some(db_name::STARTED_AT))?;
- let finished_at = env.create_database(&mut wtxn, Some(db_name::FINISHED_AT))?;
-
- let batch_status = env.create_database(&mut wtxn, Some(db_name::BATCH_STATUS))?;
- let batch_kind = env.create_database(&mut wtxn, Some(db_name::BATCH_KIND))?;
- let batch_index_tasks = env.create_database(&mut wtxn, Some(db_name::BATCH_INDEX_TASKS))?;
- let batch_enqueued_at = env.create_database(&mut wtxn, Some(db_name::BATCH_ENQUEUED_AT))?;
- let batch_started_at = env.create_database(&mut wtxn, Some(db_name::BATCH_STARTED_AT))?;
- let batch_finished_at = env.create_database(&mut wtxn, Some(db_name::BATCH_FINISHED_AT))?;
+ let features = features::FeatureData::new(&env, &mut wtxn, options.instance_features)?;
+ let queue = Queue::new(&env, &mut wtxn, &options)?;
+ let index_mapper = IndexMapper::new(&env, &mut wtxn, &options, budget)?;
wtxn.commit()?;
// allow unreachable_code to get rids of the warning in the case of a test build.
let this = Self {
- must_stop_processing: MustStopProcessing::default(),
processing_tasks: Arc::new(RwLock::new(ProcessingTasks::new())),
- file_store,
- all_tasks,
- all_batches,
- batch_to_tasks_mapping,
- // Task reverse indexes
- status,
- kind,
- index_tasks,
- canceled_by,
- enqueued_at,
- started_at,
- finished_at,
+ version,
+ queue,
+ scheduler: Scheduler::new(&options, auth_env),
- // Batch reverse indexes
- batch_status,
- batch_kind,
- batch_index_tasks,
- batch_enqueued_at,
- batch_started_at,
- batch_finished_at,
-
- index_mapper: IndexMapper::new(
- &env,
- options.indexes_path,
- budget.map_size,
- options.index_growth_amount,
- budget.index_count,
- options.enable_mdb_writemap,
- options.indexer_config,
- )?,
+ index_mapper,
env,
- // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
- wake_up: Arc::new(SignalEvent::auto(true)),
- autobatching_enabled: options.autobatching_enabled,
cleanup_enabled: options.cleanup_enabled,
- max_number_of_tasks: options.max_number_of_tasks,
- max_number_of_batched_tasks: options.max_number_of_batched_tasks,
- dumps_path: options.dumps_path,
- snapshots_path: options.snapshots_path,
- auth_path: options.auth_path,
- version_file_path: options.version_file_path,
webhook_url: options.webhook_url,
webhook_authorization_header: options.webhook_authorization_header,
embedders: Default::default(),
@@ -603,7 +297,7 @@ impl IndexScheduler {
/// Return `Ok(())` if the index scheduler is able to access one of its database.
pub fn health(&self) -> Result<()> {
let rtxn = self.env.read_txn()?;
- self.all_tasks.first(&rtxn)?;
+ self.queue.batch_to_tasks_mapping.first(&rtxn)?;
Ok(())
}
@@ -676,7 +370,7 @@ impl IndexScheduler {
}
}
- pub fn read_txn(&self) -> Result {
+ pub fn read_txn(&self) -> Result> {
self.env.read_txn().map_err(|e| e.into())
}
@@ -690,15 +384,16 @@ impl IndexScheduler {
.name(String::from("scheduler"))
.spawn(move || {
#[cfg(test)]
- run.breakpoint(Breakpoint::Init);
+ run.breakpoint(test_utils::Breakpoint::Init);
- run.wake_up.wait_timeout(std::time::Duration::from_secs(60));
+ run.scheduler.wake_up.wait_timeout(std::time::Duration::from_secs(60));
loop {
let ret = catch_unwind(AssertUnwindSafe(|| run.tick()));
match ret {
Ok(Ok(TickOutcome::TickAgain(_))) => (),
- Ok(Ok(TickOutcome::WaitForSignal)) => run.wake_up.wait(),
+ Ok(Ok(TickOutcome::WaitForSignal)) => run.scheduler.wake_up.wait(),
+ Ok(Ok(TickOutcome::StopProcessingForever)) => break,
Ok(Err(e)) => {
tracing::error!("{e}");
// Wait one second when an irrecoverable error occurs.
@@ -747,11 +442,13 @@ impl IndexScheduler {
let rtxn = self.env.read_txn()?;
self.index_mapper.index(&rtxn, name)
}
+
/// Return the boolean referring if index exists.
pub fn index_exists(&self, name: &str) -> Result {
let rtxn = self.env.read_txn()?;
self.index_mapper.index_exists(&rtxn, name)
}
+
/// Return the name of all indexes without opening them.
pub fn index_names(&self) -> Result> {
let rtxn = self.env.read_txn()?;
@@ -776,389 +473,47 @@ impl IndexScheduler {
self.index_mapper.try_for_each_index(&rtxn, f)
}
- /// Return the task ids matched by the given query from the index scheduler's point of view.
- pub(crate) fn get_task_ids(&self, rtxn: &RoTxn, query: &Query) -> Result {
- let ProcessingTasks { batch: processing_batch, processing: processing_tasks, progress: _ } =
- self.processing_tasks.read().unwrap().clone();
- let Query {
- limit,
- from,
- reverse,
- uids,
- batch_uids,
- statuses,
- types,
- index_uids,
- canceled_by,
- before_enqueued_at,
- after_enqueued_at,
- before_started_at,
- after_started_at,
- before_finished_at,
- after_finished_at,
- } = query;
-
- let mut tasks = self.all_task_ids(rtxn)?;
-
- if let Some(from) = from {
- let range = if reverse.unwrap_or_default() {
- u32::MIN..*from
- } else {
- from.saturating_add(1)..u32::MAX
- };
- tasks.remove_range(range);
- }
-
- if let Some(batch_uids) = batch_uids {
- let mut batch_tasks = RoaringBitmap::new();
- for batch_uid in batch_uids {
- if processing_batch.as_ref().map_or(false, |batch| batch.uid == *batch_uid) {
- batch_tasks |= &processing_tasks;
- } else {
- batch_tasks |= self.tasks_in_batch(rtxn, *batch_uid)?;
- }
- }
- tasks &= batch_tasks;
- }
-
- if let Some(status) = statuses {
- let mut status_tasks = RoaringBitmap::new();
- for status in status {
- match status {
- // special case for Processing tasks
- Status::Processing => {
- status_tasks |= &processing_tasks;
- }
- status => status_tasks |= &self.get_status(rtxn, *status)?,
- };
- }
- if !status.contains(&Status::Processing) {
- tasks -= &processing_tasks;
- }
- tasks &= status_tasks;
- }
-
- if let Some(uids) = uids {
- let uids = RoaringBitmap::from_iter(uids);
- tasks &= &uids;
- }
-
- if let Some(canceled_by) = canceled_by {
- let mut all_canceled_tasks = RoaringBitmap::new();
- for cancel_task_uid in canceled_by {
- if let Some(canceled_by_uid) = self.canceled_by.get(rtxn, cancel_task_uid)? {
- all_canceled_tasks |= canceled_by_uid;
- }
- }
-
- // if the canceled_by has been specified but no task
- // matches then we prefer matching zero than all tasks.
- if all_canceled_tasks.is_empty() {
- return Ok(RoaringBitmap::new());
- } else {
- tasks &= all_canceled_tasks;
- }
- }
-
- if let Some(kind) = types {
- let mut kind_tasks = RoaringBitmap::new();
- for kind in kind {
- kind_tasks |= self.get_kind(rtxn, *kind)?;
- }
- tasks &= &kind_tasks;
- }
-
- if let Some(index) = index_uids {
- let mut index_tasks = RoaringBitmap::new();
- for index in index {
- index_tasks |= self.index_tasks(rtxn, index)?;
- }
- tasks &= &index_tasks;
- }
-
- // For the started_at filter, we need to treat the part of the tasks that are processing from the part of the
- // tasks that are not processing. The non-processing ones are filtered normally while the processing ones
- // are entirely removed unless the in-memory startedAt variable falls within the date filter.
- // Once we have filtered the two subsets, we put them back together and assign it back to `tasks`.
- tasks = {
- let (mut filtered_non_processing_tasks, mut filtered_processing_tasks) =
- (&tasks - &processing_tasks, &tasks & &processing_tasks);
-
- // special case for Processing tasks
- // A closure that clears the filtered_processing_tasks if their started_at date falls outside the given bounds
- let clear_filtered_processing_tasks =
- |start: Bound, end: Bound| {
- let start = map_bound(start, |b| b.unix_timestamp_nanos());
- let end = map_bound(end, |b| b.unix_timestamp_nanos());
- let is_within_dates = RangeBounds::contains(
- &(start, end),
- &processing_batch
- .map_or_else(OffsetDateTime::now_utc, |batch| batch.started_at)
- .unix_timestamp_nanos(),
- );
- if !is_within_dates {
- filtered_processing_tasks.clear();
- }
- };
- match (after_started_at, before_started_at) {
- (None, None) => (),
- (None, Some(before)) => {
- clear_filtered_processing_tasks(Bound::Unbounded, Bound::Excluded(*before))
- }
- (Some(after), None) => {
- clear_filtered_processing_tasks(Bound::Excluded(*after), Bound::Unbounded)
- }
- (Some(after), Some(before)) => clear_filtered_processing_tasks(
- Bound::Excluded(*after),
- Bound::Excluded(*before),
- ),
- };
-
- keep_ids_within_datetimes(
- rtxn,
- &mut filtered_non_processing_tasks,
- self.started_at,
- *after_started_at,
- *before_started_at,
- )?;
- filtered_non_processing_tasks | filtered_processing_tasks
- };
-
- keep_ids_within_datetimes(
- rtxn,
- &mut tasks,
- self.enqueued_at,
- *after_enqueued_at,
- *before_enqueued_at,
- )?;
-
- keep_ids_within_datetimes(
- rtxn,
- &mut tasks,
- self.finished_at,
- *after_finished_at,
- *before_finished_at,
- )?;
-
- if let Some(limit) = limit {
- tasks = if query.reverse.unwrap_or_default() {
- tasks.into_iter().take(*limit as usize).collect()
- } else {
- tasks.into_iter().rev().take(*limit as usize).collect()
- };
- }
-
- Ok(tasks)
- }
-
- /// Return the batch ids matched by the given query from the index scheduler's point of view.
- pub(crate) fn get_batch_ids(
+ /// Returns the total number of indexes available for the specified filter.
+ /// And a `Vec` of the index_uid + its stats
+ pub fn get_paginated_indexes_stats(
&self,
- rtxn: &RoTxn,
- processing: &ProcessingTasks,
- query: &Query,
- ) -> Result {
- let Query {
- limit,
- from,
- reverse,
- uids,
- batch_uids,
- statuses,
- types,
- index_uids,
- canceled_by,
- before_enqueued_at,
- after_enqueued_at,
- before_started_at,
- after_started_at,
- before_finished_at,
- after_finished_at,
- } = query;
+ filters: &meilisearch_auth::AuthFilter,
+ from: usize,
+ limit: usize,
+ ) -> Result<(usize, Vec<(String, index_mapper::IndexStats)>)> {
+ let rtxn = self.read_txn()?;
- let mut batches = self.all_batch_ids(rtxn)?;
- if let Some(batch_id) = processing.batch.as_ref().map(|batch| batch.uid) {
- batches.insert(batch_id);
- }
+ let mut total = 0;
+ let mut iter = self
+ .index_mapper
+ .index_mapping
+ .iter(&rtxn)?
+ // in case of an error we want to keep the value to return it
+ .filter(|ret| {
+ ret.as_ref().map_or(true, |(name, _uuid)| filters.is_index_authorized(name))
+ })
+ .inspect(|_| total += 1)
+ .skip(from);
+ let ret = iter
+ .by_ref()
+ .take(limit)
+ .map(|ret| ret.map_err(Error::from))
+ .map(|ret| {
+ ret.and_then(|(name, uuid)| {
+ self.index_mapper.index_stats.get(&rtxn, &uuid).map_err(Error::from).and_then(
+ |stat| {
+ stat.map(|stat| (name.to_string(), stat))
+ .ok_or(Error::CorruptedTaskQueue)
+ },
+ )
+ })
+ })
+ .collect::>>();
- if let Some(from) = from {
- let range = if reverse.unwrap_or_default() {
- u32::MIN..*from
- } else {
- from.saturating_add(1)..u32::MAX
- };
- batches.remove_range(range);
- }
+ // We must iterate on the rest of the indexes to compute the total
+ iter.for_each(drop);
- if let Some(batch_uids) = &batch_uids {
- let batches_uids = RoaringBitmap::from_iter(batch_uids);
- batches &= batches_uids;
- }
-
- if let Some(status) = &statuses {
- let mut status_batches = RoaringBitmap::new();
- for status in status {
- match status {
- // special case for Processing batches
- Status::Processing => {
- if let Some(batch_id) = processing.batch.as_ref().map(|batch| batch.uid) {
- status_batches.insert(batch_id);
- }
- }
- // Enqueued tasks are not stored in batches
- Status::Enqueued => (),
- status => status_batches |= &self.get_batch_status(rtxn, *status)?,
- };
- }
- if !status.contains(&Status::Processing) {
- if let Some(ref batch) = processing.batch {
- batches.remove(batch.uid);
- }
- }
- batches &= status_batches;
- }
-
- if let Some(task_uids) = &uids {
- let mut batches_by_task_uids = RoaringBitmap::new();
- for task_uid in task_uids {
- if let Some(task) = self.get_task(rtxn, *task_uid)? {
- if let Some(batch_uid) = task.batch_uid {
- batches_by_task_uids.insert(batch_uid);
- }
- }
- }
- batches &= batches_by_task_uids;
- }
-
- // There is no database for this query, we must retrieve the task queried by the client and ensure it's valid
- if let Some(canceled_by) = &canceled_by {
- let mut all_canceled_batches = RoaringBitmap::new();
- for cancel_uid in canceled_by {
- if let Some(task) = self.get_task(rtxn, *cancel_uid)? {
- if task.kind.as_kind() == Kind::TaskCancelation
- && task.status == Status::Succeeded
- {
- if let Some(batch_uid) = task.batch_uid {
- all_canceled_batches.insert(batch_uid);
- }
- }
- }
- }
-
- // if the canceled_by has been specified but no batch
- // matches then we prefer matching zero than all batches.
- if all_canceled_batches.is_empty() {
- return Ok(RoaringBitmap::new());
- } else {
- batches &= all_canceled_batches;
- }
- }
-
- if let Some(kind) = &types {
- let mut kind_batches = RoaringBitmap::new();
- for kind in kind {
- kind_batches |= self.get_batch_kind(rtxn, *kind)?;
- if let Some(uid) = processing
- .batch
- .as_ref()
- .and_then(|batch| batch.kinds.contains(kind).then_some(batch.uid))
- {
- kind_batches.insert(uid);
- }
- }
- batches &= &kind_batches;
- }
-
- if let Some(index) = &index_uids {
- let mut index_batches = RoaringBitmap::new();
- for index in index {
- index_batches |= self.index_batches(rtxn, index)?;
- if let Some(uid) = processing
- .batch
- .as_ref()
- .and_then(|batch| batch.indexes.contains(index).then_some(batch.uid))
- {
- index_batches.insert(uid);
- }
- }
- batches &= &index_batches;
- }
-
- // For the started_at filter, we need to treat the part of the batches that are processing from the part of the
- // batches that are not processing. The non-processing ones are filtered normally while the processing ones
- // are entirely removed unless the in-memory startedAt variable falls within the date filter.
- // Once we have filtered the two subsets, we put them back together and assign it back to `batches`.
- batches = {
- let (mut filtered_non_processing_batches, mut filtered_processing_batches) =
- (&batches - &processing.processing, &batches & &processing.processing);
-
- // special case for Processing batches
- // A closure that clears the filtered_processing_batches if their started_at date falls outside the given bounds
- let mut clear_filtered_processing_batches =
- |start: Bound, end: Bound| {
- let start = map_bound(start, |b| b.unix_timestamp_nanos());
- let end = map_bound(end, |b| b.unix_timestamp_nanos());
- let is_within_dates = RangeBounds::contains(
- &(start, end),
- &processing
- .batch
- .as_ref()
- .map_or_else(OffsetDateTime::now_utc, |batch| batch.started_at)
- .unix_timestamp_nanos(),
- );
- if !is_within_dates {
- filtered_processing_batches.clear();
- }
- };
- match (after_started_at, before_started_at) {
- (None, None) => (),
- (None, Some(before)) => {
- clear_filtered_processing_batches(Bound::Unbounded, Bound::Excluded(*before))
- }
- (Some(after), None) => {
- clear_filtered_processing_batches(Bound::Excluded(*after), Bound::Unbounded)
- }
- (Some(after), Some(before)) => clear_filtered_processing_batches(
- Bound::Excluded(*after),
- Bound::Excluded(*before),
- ),
- };
-
- keep_ids_within_datetimes(
- rtxn,
- &mut filtered_non_processing_batches,
- self.batch_started_at,
- *after_started_at,
- *before_started_at,
- )?;
- filtered_non_processing_batches | filtered_processing_batches
- };
-
- keep_ids_within_datetimes(
- rtxn,
- &mut batches,
- self.batch_enqueued_at,
- *after_enqueued_at,
- *before_enqueued_at,
- )?;
-
- keep_ids_within_datetimes(
- rtxn,
- &mut batches,
- self.batch_finished_at,
- *after_finished_at,
- *before_finished_at,
- )?;
-
- if let Some(limit) = limit {
- batches = if query.reverse.unwrap_or_default() {
- batches.into_iter().take(*limit as usize).collect()
- } else {
- batches.into_iter().rev().take(*limit as usize).collect()
- };
- }
-
- Ok(batches)
+ ret.map(|ret| (total, ret))
}
/// The returned structure contains:
@@ -1167,39 +522,7 @@ impl IndexScheduler {
/// 3. The number of times the properties appeared.
pub fn get_stats(&self) -> Result>> {
let rtxn = self.read_txn()?;
-
- let mut res = BTreeMap::new();
-
- let processing_tasks = { self.processing_tasks.read().unwrap().processing.len() };
-
- res.insert(
- "statuses".to_string(),
- enum_iterator::all::()
- .map(|s| {
- let tasks = self.get_status(&rtxn, s)?.len();
- match s {
- Status::Enqueued => Ok((s.to_string(), tasks - processing_tasks)),
- Status::Processing => Ok((s.to_string(), processing_tasks)),
- s => Ok((s.to_string(), tasks)),
- }
- })
- .collect::>>()?,
- );
- res.insert(
- "types".to_string(),
- enum_iterator::all::()
- .map(|s| Ok((s.to_string(), self.get_kind(&rtxn, s)?.len())))
- .collect::>>()?,
- );
- res.insert(
- "indexes".to_string(),
- self.index_tasks
- .iter(&rtxn)?
- .map(|res| Ok(res.map(|(name, bitmap)| (name.to_string(), bitmap.len()))?))
- .collect::>>()?,
- );
-
- Ok(res)
+ self.queue.get_stats(&rtxn, &self.processing_tasks.read().unwrap())
}
// Return true if there is at least one task that is processing.
@@ -1212,131 +535,11 @@ impl IndexScheduler {
pub fn is_index_processing(&self, index: &str) -> Result {
let rtxn = self.env.read_txn()?;
let processing_tasks = self.processing_tasks.read().unwrap().processing.clone();
- let index_tasks = self.index_tasks(&rtxn, index)?;
+ let index_tasks = self.queue.tasks.index_tasks(&rtxn, index)?;
let nbr_index_processing_tasks = processing_tasks.intersection_len(&index_tasks);
Ok(nbr_index_processing_tasks > 0)
}
- /// Return the task ids matching the query along with the total number of tasks
- /// by ignoring the from and limit parameters from the user's point of view.
- ///
- /// There are two differences between an internal query and a query executed by
- /// the user.
- ///
- /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
- /// with many indexes internally.
- /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
- pub fn get_task_ids_from_authorized_indexes(
- &self,
- rtxn: &RoTxn,
- query: &Query,
- filters: &meilisearch_auth::AuthFilter,
- ) -> Result<(RoaringBitmap, u64)> {
- // compute all tasks matching the filter by ignoring the limits, to find the number of tasks matching
- // the filter.
- // As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares
- // us from modifying the underlying implementation, and the performance remains sufficient.
- // Should this change, we would modify `get_task_ids` to directly return the number of matching tasks.
- let total_tasks = self.get_task_ids(rtxn, &query.clone().without_limits())?;
- let mut tasks = self.get_task_ids(rtxn, query)?;
-
- // If the query contains a list of index uid or there is a finite list of authorized indexes,
- // then we must exclude all the kinds that aren't associated to one and only one index.
- if query.index_uids.is_some() || !filters.all_indexes_authorized() {
- for kind in enum_iterator::all::().filter(|kind| !kind.related_to_one_index()) {
- tasks -= self.get_kind(rtxn, kind)?;
- }
- }
-
- // Any task that is internally associated with a non-authorized index
- // must be discarded.
- if !filters.all_indexes_authorized() {
- let all_indexes_iter = self.index_tasks.iter(rtxn)?;
- for result in all_indexes_iter {
- let (index, index_tasks) = result?;
- if !filters.is_index_authorized(index) {
- tasks -= index_tasks;
- }
- }
- }
-
- Ok((tasks, total_tasks.len()))
- }
-
- /// Return the batch ids matching the query along with the total number of batches
- /// by ignoring the from and limit parameters from the user's point of view.
- ///
- /// There are two differences between an internal query and a query executed by
- /// the user.
- ///
- /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
- /// with many indexes internally.
- /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
- fn get_batch_ids_from_authorized_indexes(
- &self,
- rtxn: &RoTxn,
- processing: &ProcessingTasks,
- query: &Query,
- filters: &meilisearch_auth::AuthFilter,
- ) -> Result<(RoaringBitmap, u64)> {
- // compute all batches matching the filter by ignoring the limits, to find the number of batches matching
- // the filter.
- // As this causes us to compute the filter twice it is slightly inefficient, but doing it this way spares
- // us from modifying the underlying implementation, and the performance remains sufficient.
- // Should this change, we would modify `get_batch_ids` to directly return the number of matching batches.
- let total_batches =
- self.get_batch_ids(rtxn, processing, &query.clone().without_limits())?;
- let mut batches = self.get_batch_ids(rtxn, processing, query)?;
-
- // If the query contains a list of index uid or there is a finite list of authorized indexes,
- // then we must exclude all the batches that only contains tasks associated to multiple indexes.
- // This works because we don't autobatch tasks associated to multiple indexes with tasks associated
- // to a single index. e.g: IndexSwap cannot be batched with IndexCreation.
- if query.index_uids.is_some() || !filters.all_indexes_authorized() {
- for kind in enum_iterator::all::().filter(|kind| !kind.related_to_one_index()) {
- batches -= self.get_kind(rtxn, kind)?;
- if let Some(batch) = processing.batch.as_ref() {
- if batch.kinds.contains(&kind) {
- batches.remove(batch.uid);
- }
- }
- }
- }
-
- // Any batch that is internally associated with at least one authorized index
- // must be returned.
- if !filters.all_indexes_authorized() {
- let mut valid_indexes = RoaringBitmap::new();
- let mut forbidden_indexes = RoaringBitmap::new();
-
- let all_indexes_iter = self.batch_index_tasks.iter(rtxn)?;
- for result in all_indexes_iter {
- let (index, index_tasks) = result?;
- if filters.is_index_authorized(index) {
- valid_indexes |= index_tasks;
- } else {
- forbidden_indexes |= index_tasks;
- }
- }
- if let Some(batch) = processing.batch.as_ref() {
- for index in &batch.indexes {
- if filters.is_index_authorized(index) {
- valid_indexes.insert(batch.uid);
- } else {
- forbidden_indexes.insert(batch.uid);
- }
- }
- }
-
- // If a batch had ONE valid task then it should be returned
- let invalid_batches = forbidden_indexes - valid_indexes;
-
- batches -= invalid_batches;
- }
-
- Ok((batches, total_batches.len()))
- }
-
/// Return the tasks matching the query from the user's point of view along
/// with the total number of tasks matching the query, ignoring from and limit.
///
@@ -1344,53 +547,35 @@ impl IndexScheduler {
/// the user.
///
/// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
- /// with many indexes internally.
+ /// with many indexes internally.
/// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
pub fn get_tasks_from_authorized_indexes(
&self,
- query: Query,
+ query: &Query,
filters: &meilisearch_auth::AuthFilter,
) -> Result<(Vec, u64)> {
- let rtxn = self.env.read_txn()?;
+ let rtxn = self.read_txn()?;
+ let processing = self.processing_tasks.read().unwrap();
+ self.queue.get_tasks_from_authorized_indexes(&rtxn, query, filters, &processing)
+ }
- let (tasks, total) = self.get_task_ids_from_authorized_indexes(&rtxn, &query, filters)?;
- let tasks = if query.reverse.unwrap_or_default() {
- Box::new(tasks.into_iter()) as Box>
- } else {
- Box::new(tasks.into_iter().rev()) as Box>
- };
- let tasks =
- self.get_existing_tasks(&rtxn, tasks.take(query.limit.unwrap_or(u32::MAX) as usize))?;
-
- let ProcessingTasks { batch, processing, progress } =
- self.processing_tasks.read().map_err(|_| Error::CorruptedTaskQueue)?.clone();
-
- // ignored for now, might be added to batch details later
- let _ = progress;
-
- let ret = tasks.into_iter();
- if processing.is_empty() || batch.is_none() {
- Ok((ret.collect(), total))
- } else {
- // Safe because we ensured there was a batch in the previous branch
- let batch = batch.unwrap();
- Ok((
- ret.map(|task| {
- if processing.contains(task.uid) {
- Task {
- status: Status::Processing,
- batch_uid: Some(batch.uid),
- started_at: Some(batch.started_at),
- ..task
- }
- } else {
- task
- }
- })
- .collect(),
- total,
- ))
- }
+ /// Return the task ids matching the query along with the total number of tasks
+ /// by ignoring the from and limit parameters from the user's point of view.
+ ///
+ /// There are two differences between an internal query and a query executed by
+ /// the user.
+ ///
+ /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
+ /// with many indexes internally.
+ /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
+ pub fn get_task_ids_from_authorized_indexes(
+ &self,
+ query: &Query,
+ filters: &meilisearch_auth::AuthFilter,
+ ) -> Result<(RoaringBitmap, u64)> {
+ let rtxn = self.read_txn()?;
+ let processing = self.processing_tasks.read().unwrap();
+ self.queue.get_task_ids_from_authorized_indexes(&rtxn, query, filters, &processing)
}
/// Return the batches matching the query from the user's point of view along
@@ -1400,31 +585,35 @@ impl IndexScheduler {
/// the user.
///
/// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
- /// with many indexes internally.
+ /// with many indexes internally.
/// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
pub fn get_batches_from_authorized_indexes(
&self,
- query: Query,
+ query: &Query,
filters: &meilisearch_auth::AuthFilter,
) -> Result<(Vec, u64)> {
- let rtxn = self.env.read_txn()?;
- let processing = self.processing_tasks.read().unwrap().clone();
+ let rtxn = self.read_txn()?;
+ let processing = self.processing_tasks.read().unwrap();
+ self.queue.get_batches_from_authorized_indexes(&rtxn, query, filters, &processing)
+ }
- let (batches, total) =
- self.get_batch_ids_from_authorized_indexes(&rtxn, &processing, &query, filters)?;
- let batches = if query.reverse.unwrap_or_default() {
- Box::new(batches.into_iter()) as Box>
- } else {
- Box::new(batches.into_iter().rev()) as Box>
- };
-
- let batches = self.get_existing_batches(
- &rtxn,
- &processing,
- batches.take(query.limit.unwrap_or(u32::MAX) as usize),
- )?;
-
- Ok((batches, total))
+ /// Return the batch ids matching the query along with the total number of batches
+ /// by ignoring the from and limit parameters from the user's point of view.
+ ///
+ /// There are two differences between an internal query and a query executed by
+ /// the user.
+ ///
+ /// 1. IndexSwap tasks are not publicly associated with any index, but they are associated
+ /// with many indexes internally.
+ /// 2. The user may not have the rights to access the tasks (internally) associated with all indexes.
+ pub fn get_batch_ids_from_authorized_indexes(
+ &self,
+ query: &Query,
+ filters: &meilisearch_auth::AuthFilter,
+ ) -> Result<(RoaringBitmap, u64)> {
+ let rtxn = self.read_txn()?;
+ let processing = self.processing_tasks.read().unwrap();
+ self.queue.get_batch_ids_from_authorized_indexes(&rtxn, query, filters, &processing)
}
/// Register a new task in the scheduler.
@@ -1436,73 +625,15 @@ impl IndexScheduler {
task_id: Option