Merge #4912

4912: Allow Meilitool to dumplessly, offline upgrade v1.9 -> v1.10 in some conditions r=Kerollmops a=dureuill

- bail early if the DB contains at least 1 REST embedder, providing the list of detected REST embedders, and without modifying the DB
- Might depend on the feature set that meilitool was compiled with and the featureset that the Meilisearch that created the DB was compiled with 💀. In case of runtime error, try again with a different feature set (passing or not passing `-p meilitool` when building after a `cargo clean`)

Co-authored-by: Louis Dureuil <louis@meilisearch.com>

Cargo.lock

@@ -471,7 +471,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "benchmarks"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "bytes",
@@ -652,7 +652,7 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "time",
@@ -1622,7 +1622,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1834,7 +1834,7 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
 
 [[package]]
 name = "file-store"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "tempfile",
  "thiserror",
@@ -1856,7 +1856,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "insta",
  "nom",
@@ -1876,7 +1876,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2000,7 +2000,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "arbitrary",
  "clap",
@@ -2552,7 +2552,7 @@ checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
 
 [[package]]
 name = "index-scheduler"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "arroy",
@@ -2746,7 +2746,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -3365,7 +3365,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "insta",
  "md5",
@@ -3374,7 +3374,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -3463,7 +3463,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "base64 0.22.1",
  "enum-iterator",
@@ -3482,7 +3482,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -3512,7 +3512,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -3520,6 +3520,7 @@ dependencies = [
  "file-store",
  "meilisearch-auth",
  "meilisearch-types",
+ "serde",
  "time",
  "uuid",
 ]
@@ -3542,7 +3543,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "arroy",
  "big_s",
@@ -3976,7 +3977,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "big_s",
  "serde_json",
@@ -4834,9 +4835,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
@@ -4852,9 +4853,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6361,7 +6362,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.10.0"
+version = "1.10.1"
 dependencies = [
  "anyhow",
  "build-info",

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.10.0"
+version = "1.10.1"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

index-scheduler/src/autobatcher.rs

@@ -25,8 +25,9 @@ enum AutobatchKind {
         primary_key: Option<String>,
     },
     DocumentEdition,
-    DocumentDeletion,
-    DocumentDeletionByFilter,
+    DocumentDeletion {
+        by_filter: bool,
+    },
     DocumentClear,
     Settings {
         allow_index_creation: bool,
@@ -65,10 +66,12 @@ impl From<KindWithContent> for AutobatchKind {
                 ..
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
             KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
-            KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
+            KindWithContent::DocumentDeletion { .. } => {
+                AutobatchKind::DocumentDeletion { by_filter: false }
+            }
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
             KindWithContent::DocumentDeletionByFilter { .. } => {
-                AutobatchKind::DocumentDeletionByFilter
+                AutobatchKind::DocumentDeletion { by_filter: true }
             }
             KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
                 AutobatchKind::Settings {
@@ -105,9 +108,7 @@ pub enum BatchKind {
     },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
-    },
-    DocumentDeletionByFilter {
-        id: TaskId,
+        includes_by_filter: bool,
     },
     ClearAndSettings {
         other: Vec<TaskId>,
@@ -205,12 +206,13 @@ impl BatchKind {
                 allow_index_creation,
             ),
             K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
-            K::DocumentDeletion => {
-                (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
-            }
-            K::DocumentDeletionByFilter => {
-                (Break(BatchKind::DocumentDeletionByFilter { id: task_id }), false)
-            }
+            K::DocumentDeletion { by_filter: includes_by_filter } => (
+                Continue(BatchKind::DocumentDeletion {
+                    deletion_ids: vec![task_id],
+                    includes_by_filter,
+                }),
+                false,
+            ),
             K::Settings { allow_index_creation } => (
                 Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
                 allow_index_creation,
@@ -228,7 +230,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition | K::DocumentDeletionByFilter) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -264,7 +266,7 @@ impl BatchKind {
             // The index deletion can batch with everything but must stop after
             (
                 BatchKind::DocumentClear { mut ids }
-                | BatchKind::DocumentDeletion { deletion_ids: mut ids }
+                | BatchKind::DocumentDeletion { deletion_ids: mut ids, includes_by_filter: _ }
                 | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
                 | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
                 K::IndexDeletion,
@@ -284,7 +286,7 @@ impl BatchKind {
 
             (
                 BatchKind::DocumentClear { mut ids },
-                K::DocumentClear | K::DocumentDeletion,
+                K::DocumentClear | K::DocumentDeletion { by_filter: _ },
             ) => {
                 ids.push(id);
                 Continue(BatchKind::DocumentClear { ids })
@@ -328,7 +330,7 @@ impl BatchKind {
             }
             (
                 BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
-                K::DocumentDeletion,
+                K::DocumentDeletion { by_filter: false },
             ) => {
                 operation_ids.push(id);
 
@@ -339,6 +341,13 @@ impl BatchKind {
                     operation_ids,
                 })
             }
+            // We can't batch a document operation with a delete by filter
+            (
+                this @ BatchKind::DocumentOperation { .. },
+                K::DocumentDeletion { by_filter: true },
+            ) => {
+                Break(this)
+            }
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
@@ -357,13 +366,18 @@ impl BatchKind {
                 operation_ids,
             }),
 
-            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentClear) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: _ }, K::DocumentClear) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
+            // we can't autobatch the deletion and import if the document deletion contained a filter
+            (
+                this @ BatchKind::DocumentDeletion { deletion_ids: _, includes_by_filter: true },
+                K::DocumentImport { .. }
+            ) => Break(this),
             // we can autobatch the deletion and import if the index already exists
             (
-                BatchKind::DocumentDeletion { mut deletion_ids },
+                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if index_already_exists => {
                 deletion_ids.push(id);
@@ -377,7 +391,7 @@ impl BatchKind {
             }
             // we can autobatch the deletion and import if both can't create an index
             (
-                BatchKind::DocumentDeletion { mut deletion_ids },
+                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if !allow_index_creation => {
                 deletion_ids.push(id);
@@ -396,9 +410,9 @@ impl BatchKind {
             ) => {
                 Break(this)
             }
-            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentDeletion) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter }, K::DocumentDeletion { by_filter }) => {
                 deletion_ids.push(id);
-                Continue(BatchKind::DocumentDeletion { deletion_ids })
+                Continue(BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: includes_by_filter | by_filter })
             }
             (this @ BatchKind::DocumentDeletion { .. }, K::Settings { .. }) => Break(this),
 
@@ -412,7 +426,7 @@ impl BatchKind {
             }),
             (
                 this @ BatchKind::Settings { .. },
-                K::DocumentImport { .. } | K::DocumentDeletion,
+                K::DocumentImport { .. } | K::DocumentDeletion { .. },
             ) => Break(this),
             (
                 BatchKind::Settings { mut settings_ids, allow_index_creation },
@@ -443,7 +457,7 @@ impl BatchKind {
                     settings_ids,
                     allow_index_creation,
                 },
-                K::DocumentDeletion,
+                K::DocumentDeletion { .. },
             ) => {
                 other.push(id);
                 Continue(BatchKind::ClearAndSettings {
@@ -505,7 +519,7 @@ impl BatchKind {
             // this MUST be AFTER the two previous branch
             (
                 this @ BatchKind::SettingsAndDocumentOperation { .. },
-                K::DocumentDeletion | K::DocumentImport { .. },
+                K::DocumentDeletion { .. } | K::DocumentImport { .. },
             ) => Break(this),
             (
                 BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
@@ -525,8 +539,7 @@ impl BatchKind {
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
                 | BatchKind::IndexSwap { .. }
-                | BatchKind::DocumentEdition { .. }
-                | BatchKind::DocumentDeletionByFilter { .. },
+                | BatchKind::DocumentEdition { .. },
                 _,
             ) => {
                 unreachable!()
@@ -616,6 +629,13 @@ mod tests {
         }
     }
 
+    fn doc_del_fil() -> KindWithContent {
+        KindWithContent::DocumentDeletionByFilter {
+            index_uid: String::from("doggo"),
+            filter_expr: serde_json::json!("cuteness > 100"),
+        }
+    }
+
     fn doc_clr() -> KindWithContent {
         KindWithContent::DocumentClear { index_uid: String::from("doggo") }
     }
@@ -676,10 +696,16 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple DocumentDeletion together
-        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
+
+        // we can autobatch one or multiple DocumentDeletionByFilter together
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
 
         // we can autobatch one or multiple Settings together
         debug_snapshot!(autobatch_from(true, None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
@@ -722,25 +748,63 @@ mod tests {
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
+
+        // But we can't autobatch document addition with document deletion by filter
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        // And the other way around
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
     fn simple_document_operation_dont_autobatch_with_other() {
-        // addition, updates and deletion can't batch together
+        // addition, updates and deletion by filter can't batch together
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
@@ -807,6 +871,7 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -816,6 +881,7 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -827,6 +893,7 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(true, None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -836,6 +903,7 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -901,10 +969,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
 
         // batch deletion and addition
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
     }
 
     #[test]

index-scheduler/src/batch.rs

@@ -110,9 +110,9 @@ pub(crate) enum IndexOperation {
         index_uid: String,
         task: Task,
     },
-    IndexDocumentDeletionByFilter {
+    DocumentDeletion {
         index_uid: String,
-        task: Task,
+        tasks: Vec<Task>,
     },
     DocumentClear {
         index_uid: String,
@@ -165,11 +165,11 @@ impl Batch {
             Batch::IndexOperation { op, .. } => match op {
                 IndexOperation::DocumentOperation { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
+                | IndexOperation::DocumentDeletion { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
                 }
-                IndexOperation::DocumentEdition { task, .. }
-                | IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
+                IndexOperation::DocumentEdition { task, .. } => {
                     RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
                 }
                 IndexOperation::SettingsAndDocumentOperation {
@@ -234,7 +234,7 @@ impl IndexOperation {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
             | IndexOperation::DocumentEdition { index_uid, .. }
-            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
+            | IndexOperation::DocumentDeletion { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -252,8 +252,8 @@ impl fmt::Display for IndexOperation {
             IndexOperation::DocumentEdition { .. } => {
                 f.write_str("IndexOperation::DocumentEdition")
             }
-            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
-                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
+            IndexOperation::DocumentDeletion { .. } => {
+                f.write_str("IndexOperation::DocumentDeletion")
             }
             IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
             IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
@@ -289,21 +289,6 @@ impl IndexScheduler {
                 },
                 must_create_index,
             })),
-            BatchKind::DocumentDeletionByFilter { id } => {
-                let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
-                match &task.kind {
-                    KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
-                        Ok(Some(Batch::IndexOperation {
-                            op: IndexOperation::IndexDocumentDeletionByFilter {
-                                index_uid: index_uid.clone(),
-                                task,
-                            },
-                            must_create_index: false,
-                        }))
-                    }
-                    _ => unreachable!(),
-                }
-            }
             BatchKind::DocumentEdition { id } => {
                 let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 match &task.kind {
@@ -366,30 +351,11 @@ impl IndexScheduler {
                     must_create_index,
                 }))
             }
-            BatchKind::DocumentDeletion { deletion_ids } => {
+            BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: _ } => {
                 let tasks = self.get_existing_tasks(rtxn, deletion_ids)?;
 
-                let mut operations = Vec::with_capacity(tasks.len());
-                let mut documents_counts = Vec::with_capacity(tasks.len());
-                for task in &tasks {
-                    match task.kind {
-                        KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
-                            operations.push(DocumentOperation::Delete(documents_ids.clone()));
-                            documents_counts.push(documents_ids.len() as u64);
-                        }
-                        _ => unreachable!(),
-                    }
-                }
-
                 Ok(Some(Batch::IndexOperation {
-                    op: IndexOperation::DocumentOperation {
-                        index_uid,
-                        primary_key: None,
-                        method: IndexDocumentsMethod::ReplaceDocuments,
-                        documents_counts,
-                        operations,
-                        tasks,
-                    },
+                    op: IndexOperation::DocumentDeletion { index_uid, tasks },
                     must_create_index,
                 }))
             }
@@ -1439,7 +1405,7 @@ impl IndexScheduler {
                 {
                     (original_filter, context, function)
                 } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
+                    // In the case of a `documentEdition` the details MUST be set
                     unreachable!();
                 };
 
@@ -1469,52 +1435,102 @@ impl IndexScheduler {
 
                 Ok(vec![task])
             }
-            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
-                let filter =
-                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
-                        &task.kind
-                    {
-                        filter_expr
-                    } else {
-                        unreachable!()
-                    };
-                let deleted_documents = delete_document_by_filter(
-                    index_wtxn,
-                    filter,
-                    self.index_mapper.indexer_config(),
-                    self.must_stop_processing.clone(),
-                    index,
-                );
-                let original_filter = if let Some(Details::DocumentDeletionByFilter {
-                    original_filter,
-                    deleted_documents: _,
-                }) = task.details
-                {
-                    original_filter
-                } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
-                    unreachable!();
-                };
+            IndexOperation::DocumentDeletion { mut tasks, index_uid: _ } => {
+                let mut to_delete = RoaringBitmap::new();
+                let external_documents_ids = index.external_documents_ids();
 
-                match deleted_documents {
-                    Ok(deleted_documents) => {
-                        task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(deleted_documents),
-                        });
-                    }
-                    Err(e) => {
-                        task.status = Status::Failed;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(0),
-                        });
-                        task.error = Some(e.into());
+                for task in tasks.iter_mut() {
+                    let before = to_delete.len();
+                    task.status = Status::Succeeded;
+
+                    match &task.kind {
+                        KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
+                            for id in documents_ids {
+                                if let Some(id) = external_documents_ids.get(index_wtxn, id)? {
+                                    to_delete.insert(id);
+                                }
+                            }
+                            let will_be_removed = to_delete.len() - before;
+                            task.details = Some(Details::DocumentDeletion {
+                                provided_ids: documents_ids.len(),
+                                deleted_documents: Some(will_be_removed),
+                            });
+                        }
+                        KindWithContent::DocumentDeletionByFilter { index_uid: _, filter_expr } => {
+                            let before = to_delete.len();
+                            let filter = match Filter::from_json(filter_expr) {
+                                Ok(filter) => filter,
+                                Err(err) => {
+                                    // theorically, this should be catched by deserr before reaching the index-scheduler and cannot happens
+                                    task.status = Status::Failed;
+                                    task.error = match err {
+                                        milli::Error::UserError(
+                                            milli::UserError::InvalidFilterExpression { .. },
+                                        ) => Some(
+                                            Error::from(err)
+                                                .with_custom_error_code(Code::InvalidDocumentFilter)
+                                                .into(),
+                                        ),
+                                        e => Some(e.into()),
+                                    };
+                                    None
+                                }
+                            };
+                            if let Some(filter) = filter {
+                                let candidates =
+                                    filter.evaluate(index_wtxn, index).map_err(|err| match err {
+                                        milli::Error::UserError(
+                                            milli::UserError::InvalidFilter(_),
+                                        ) => Error::from(err)
+                                            .with_custom_error_code(Code::InvalidDocumentFilter),
+                                        e => e.into(),
+                                    });
+                                match candidates {
+                                    Ok(candidates) => to_delete |= candidates,
+                                    Err(err) => {
+                                        task.status = Status::Failed;
+                                        task.error = Some(err.into());
+                                    }
+                                };
+                            }
+                            let will_be_removed = to_delete.len() - before;
+                            if let Some(Details::DocumentDeletionByFilter {
+                                original_filter: _,
+                                deleted_documents,
+                            }) = &mut task.details
+                            {
+                                *deleted_documents = Some(will_be_removed);
+                            } else {
+                                // In the case of a `documentDeleteByFilter` the details MUST be set
+                                unreachable!()
+                            }
+                        }
+                        _ => unreachable!(),
                     }
                 }
 
-                Ok(vec![task])
+                let config = IndexDocumentsConfig {
+                    update_method: IndexDocumentsMethod::ReplaceDocuments,
+                    ..Default::default()
+                };
+
+                let must_stop_processing = self.must_stop_processing.clone();
+                let mut builder = milli::update::IndexDocuments::new(
+                    index_wtxn,
+                    index,
+                    self.index_mapper.indexer_config(),
+                    config,
+                    |indexing_step| tracing::debug!(update = ?indexing_step),
+                    || must_stop_processing.get(),
+                )?;
+
+                let (new_builder, _count) =
+                    builder.remove_documents_from_db_no_batch(&to_delete)?;
+                builder = new_builder;
+
+                let _ = builder.execute()?;
+
+                Ok(tasks)
             }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
@@ -1718,46 +1734,6 @@ impl IndexScheduler {
     }
 }
 
-fn delete_document_by_filter<'a>(
-    wtxn: &mut RwTxn<'a>,
-    filter: &serde_json::Value,
-    indexer_config: &IndexerConfig,
-    must_stop_processing: MustStopProcessing,
-    index: &'a Index,
-) -> Result<u64> {
-    let filter = Filter::from_json(filter)?;
-    Ok(if let Some(filter) = filter {
-        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
-            milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
-                Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
-            }
-            e => e.into(),
-        })?;
-
-        let config = IndexDocumentsConfig {
-            update_method: IndexDocumentsMethod::ReplaceDocuments,
-            ..Default::default()
-        };
-
-        let mut builder = milli::update::IndexDocuments::new(
-            wtxn,
-            index,
-            indexer_config,
-            config,
-            |indexing_step| tracing::debug!(update = ?indexing_step),
-            || must_stop_processing.get(),
-        )?;
-
-        let (new_builder, count) = builder.remove_documents_from_db_no_batch(&candidates)?;
-        builder = new_builder;
-
-        let _ = builder.execute()?;
-        count
-    } else {
-        0
-    })
-}
-
 fn edit_documents_by_function<'a>(
     wtxn: &mut RwTxn<'a>,
     filter: &Option<serde_json::Value>,

index-scheduler/src/insta_snapshot.rs

@@ -11,6 +11,9 @@ use crate::index_mapper::IndexMapper;
 use crate::{IndexScheduler, Kind, Status, BEI128};
 
 pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
+    // Since we'll snapshot the index right afterward, we don't need to ensure it's internally consistent for every run.
+    // We can only do it for the release run, where the function runs way faster.
+    #[cfg(not(debug_assertions))]
     scheduler.assert_internally_consistent();
 
     let IndexScheduler {

index-scheduler/src/lib.rs

@@ -35,6 +35,7 @@ pub type TaskId = u32;
 use std::collections::{BTreeMap, HashMap};
 use std::io::{self, BufReader, Read};
 use std::ops::{Bound, RangeBounds};
+use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::Ordering::{self, Relaxed};
 use std::sync::atomic::{AtomicBool, AtomicU32};
@@ -612,19 +613,24 @@ impl IndexScheduler {
                 #[cfg(test)]
                 run.breakpoint(Breakpoint::Init);
 
-                run.wake_up.wait();
+                run.wake_up.wait_timeout(std::time::Duration::from_secs(60));
 
                 loop {
-                    match run.tick() {
-                        Ok(TickOutcome::TickAgain(_)) => (),
-                        Ok(TickOutcome::WaitForSignal) => run.wake_up.wait(),
-                        Err(e) => {
+                    let ret = catch_unwind(AssertUnwindSafe(|| run.tick()));
+                    match ret {
+                        Ok(Ok(TickOutcome::TickAgain(_))) => (),
+                        Ok(Ok(TickOutcome::WaitForSignal)) => run.wake_up.wait(),
+                        Ok(Err(e)) => {
                             tracing::error!("{e}");
                             // Wait one second when an irrecoverable error occurs.
                             if !e.is_recoverable() {
                                 std::thread::sleep(Duration::from_secs(1));
                             }
                         }
+                        Err(_panic) => {
+                            tracing::error!("Internal error: Unexpected panic in the `IndexScheduler::run` method.");
+
+                        }
                     }
                 }
             })
@@ -1758,6 +1764,7 @@ mod tests {
     use crossbeam::channel::RecvTimeoutError;
     use file_store::File;
     use insta::assert_json_snapshot;
+    use maplit::btreeset;
     use meili_snap::{json_string, snapshot};
     use meilisearch_auth::AuthFilter;
     use meilisearch_types::document_formats::DocumentFormatError;
@@ -2000,11 +2007,13 @@ mod tests {
         fn advance_till(&mut self, breakpoints: impl IntoIterator<Item = Breakpoint>) {
             for breakpoint in breakpoints {
                 let b = self.advance();
-                let state = snapshot_index_scheduler(&self.index_scheduler);
                 assert_eq!(
-                    b, breakpoint,
-                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{state}",
-                    breakpoint, b
+                    b,
+                    breakpoint,
+                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{}",
+                    breakpoint,
+                    b,
+                    snapshot_index_scheduler(&self.index_scheduler)
                 );
             }
         }
@@ -2028,7 +2037,6 @@ mod tests {
         // Wait for one successful batch.
         #[track_caller]
         fn advance_one_successful_batch(&mut self) {
-            self.index_scheduler.assert_internally_consistent();
             self.advance_till([Start, BatchCreated]);
             loop {
                 match self.advance() {
@@ -2047,7 +2055,6 @@ mod tests {
             }
 
             self.advance_till([AfterProcessing]);
-            self.index_scheduler.assert_internally_consistent();
         }
 
         // Wait for one failed batch.
@@ -2547,6 +2554,117 @@ mod tests {
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
 
+    #[test]
+    fn fail_in_process_batch_for_document_deletion() {
+        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+        use meilisearch_types::settings::{Settings, Unchecked};
+        let mut new_settings: Box<Settings<Unchecked>> = Box::default();
+        new_settings.filterable_attributes = Setting::Set(btreeset!(S("catto")));
+
+        index_scheduler
+            .register(
+                KindWithContent::SettingsUpdate {
+                    index_uid: S("doggos"),
+                    new_settings,
+                    is_deletion: false,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+
+        let content = r#"[
+            { "id": 1, "doggo": "jean bob" },
+            { "id": 2, "catto": "jorts" },
+            { "id": 3, "doggo": "bork" }
+        ]"#;
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
+        let documents_count = read_json(content.as_bytes(), &mut file).unwrap();
+        file.persist().unwrap();
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: Some(S("id")),
+                    method: ReplaceDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_setting_and_document_addition");
+
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_settings");
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_documents");
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletion {
+                    index_uid: S("doggos"),
+                    documents_ids: vec![S("1")],
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        // This one should not be catched by Meilisearch but it's still nice to handle it because if one day we break the filters it could happens
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!(true),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        // Should fail because the ids are not filterable
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!("id = 2"),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!("catto EXISTS"),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_document_deletions");
+
+        // Everything should be batched together
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_documents");
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents_remaining_should_only_be_bork");
+    }
+
     #[test]
     fn do_not_batch_task_of_different_indexes() {
         let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
@@ -5397,6 +5515,9 @@ mod tests {
                     ),
                     prompt: PromptData {
                         template: "{{doc.doggo}}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[1, 2]>,
@@ -5611,7 +5732,10 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[0]>,
@@ -5651,7 +5775,10 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[]>,

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap

@@ -0,0 +1,44 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap

@@ -0,0 +1,43 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [1,]
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap

@@ -0,0 +1,43 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [1,]
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap

@@ -0,0 +1,56 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+2 {uid: 2, status: succeeded, details: { received_document_ids: 1, deleted_documents: Some(1) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
+3 {uid: 3, status: failed, error: ResponseError { code: 200, message: "Invalid type for filter subexpression: expected: String, Array, found: true.", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: true, deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
+4 {uid: 4, status: failed, error: ResponseError { code: 200, message: "Attribute `id` is not filterable. Available filterable attributes are: `catto`.\n1:3 id = 2", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: "id = 2", deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
+5 {uid: 5, status: succeeded, details: { original_filter: "catto EXISTS", deleted_documents: Some(1) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,2,5,]
+failed [3,4,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"documentDeletion" [2,3,4,5,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,3,4,5,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+[timestamp] [4,]
+[timestamp] [5,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,3,4,5,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,3,4,5,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap

@@ -0,0 +1,9 @@
+---
+source: index-scheduler/src/lib.rs
+---
+[
+  {
+    "id": 3,
+    "doggo": "bork"
+  }
+]

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap

@@ -0,0 +1,53 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+2 {uid: 2, status: enqueued, details: { received_document_ids: 1, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
+3 {uid: 3, status: enqueued, details: { original_filter: true, deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
+4 {uid: 4, status: enqueued, details: { original_filter: "id = 2", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
+5 {uid: 5, status: enqueued, details: { original_filter: "catto EXISTS", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
+----------------------------------------------------------------------
+### Status:
+enqueued [2,3,4,5,]
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"documentDeletion" [2,3,4,5,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,3,4,5,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+[timestamp] [4,]
+[timestamp] [5,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap

@@ -0,0 +1,39 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

meilisearch-types/src/error.rs

@@ -388,6 +388,7 @@ impl ErrorCode for milli::Error {
                     | UserError::InvalidOpenAiModelDimensionsMax { .. }
                     | UserError::InvalidSettingsDimensions { .. }
                     | UserError::InvalidUrl { .. }
+                    | UserError::InvalidSettingsDocumentTemplateMaxBytes { .. }
                     | UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders,
                     UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
                     UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,

meilisearch-types/src/versioning.rs

@@ -10,38 +10,52 @@ static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
 static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
 
 /// Persists the version of the current Meilisearch binary to a VERSION file
-pub fn create_version_file(db_path: &Path) -> io::Result<()> {
+pub fn create_current_version_file(db_path: &Path) -> io::Result<()> {
+    create_version_file(db_path, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
+}
+
+pub fn create_version_file(
+    db_path: &Path,
+    major: &str,
+    minor: &str,
+    patch: &str,
+) -> io::Result<()> {
     let version_path = db_path.join(VERSION_FILE_NAME);
-    fs::write(version_path, format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH))
+    fs::write(version_path, format!("{}.{}.{}", major, minor, patch))
 }
 
 /// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
 pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
-    let version_path = db_path.join(VERSION_FILE_NAME);
+    let (major, minor, patch) = get_version(db_path)?;
 
-    match fs::read_to_string(version_path) {
-        Ok(version) => {
-            let version_components = version.split('.').collect::<Vec<_>>();
-            let (major, minor, patch) = match &version_components[..] {
-                [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
-                _ => return Err(VersionFileError::MalformedVersionFile.into()),
-            };
-
-            if major != VERSION_MAJOR || minor != VERSION_MINOR {
-                return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
-            }
-        }
-        Err(error) => {
-            return match error.kind() {
-                ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()),
-                _ => Err(error.into()),
-            }
-        }
+    if major != VERSION_MAJOR || minor != VERSION_MINOR {
+        return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
     }
 
     Ok(())
 }
 
+pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> {
+    let version_path = db_path.join(VERSION_FILE_NAME);
+
+    match fs::read_to_string(version_path) {
+        Ok(version) => parse_version(&version),
+        Err(error) => match error.kind() {
+            ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile),
+            _ => Err(error.into()),
+        },
+    }
+}
+
+pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> {
+    let version_components = version.split('.').collect::<Vec<_>>();
+    let (major, minor, patch) = match &version_components[..] {
+        [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
+        _ => return Err(VersionFileError::MalformedVersionFile),
+    };
+    Ok((major, minor, patch))
+}
+
 #[derive(thiserror::Error, Debug)]
 pub enum VersionFileError {
     #[error(
@@ -58,4 +72,7 @@ pub enum VersionFileError {
         env!("CARGO_PKG_VERSION").to_string()
     )]
     VersionMismatch { major: String, minor: String, patch: String },
+
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
 }

meilisearch/src/lib.rs

@@ -13,11 +13,10 @@ pub mod search_queue;
 
 use std::fs::File;
 use std::io::{BufReader, BufWriter};
-use std::num::NonZeroUsize;
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::thread::{self, available_parallelism};
+use std::thread;
 use std::time::Duration;
 
 use actix_cors::Cors;
@@ -37,7 +36,7 @@ use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchR
 use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::KindWithContent;
-use meilisearch_types::versioning::{check_version_file, create_version_file};
+use meilisearch_types::versioning::{check_version_file, create_current_version_file};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
@@ -118,6 +117,7 @@ pub type LogStderrType = tracing_subscriber::filter::Filtered<
 pub fn create_app(
     index_scheduler: Data<IndexScheduler>,
     auth_controller: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
     opt: Opt,
     logs: (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
@@ -137,6 +137,7 @@ pub fn create_app(
                 s,
                 index_scheduler.clone(),
                 auth_controller.clone(),
+                search_queue.clone(),
                 &opt,
                 logs,
                 analytics.clone(),
@@ -318,7 +319,7 @@ fn open_or_create_database_unchecked(
     match (
         index_scheduler_builder(),
         auth_controller.map_err(anyhow::Error::from),
-        create_version_file(&opt.db_path).map_err(anyhow::Error::from),
+        create_current_version_file(&opt.db_path).map_err(anyhow::Error::from),
     ) {
         (Ok(i), Ok(a), Ok(())) => Ok((i, a)),
         (Err(e), _, _) | (_, Err(e), _) | (_, _, Err(e)) => {
@@ -469,19 +470,16 @@ pub fn configure_data(
     config: &mut web::ServiceConfig,
     index_scheduler: Data<IndexScheduler>,
     auth: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
     opt: &Opt,
     (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
 ) {
-    let search_queue = SearchQueue::new(
-        opt.experimental_search_queue_size,
-        available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
-    );
     let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)
-        .app_data(web::Data::new(search_queue))
+        .app_data(search_queue)
         .app_data(web::Data::from(analytics))
         .app_data(web::Data::new(logs_route))
         .app_data(web::Data::new(logs_stderr))

meilisearch/src/main.rs

@@ -1,8 +1,10 @@
 use std::env;
 use std::io::{stderr, LineWriter, Write};
+use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::thread::available_parallelism;
 
 use actix_web::http::KeepAlive;
 use actix_web::web::Data;
@@ -11,6 +13,7 @@ use index_scheduler::IndexScheduler;
 use is_terminal::IsTerminal;
 use meilisearch::analytics::Analytics;
 use meilisearch::option::LogMode;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{
     analytics, create_app, setup_meilisearch, LogRouteHandle, LogRouteType, LogStderrHandle,
     LogStderrType, Opt, SubscriberForSecondLayer,
@@ -148,11 +151,17 @@ async fn run_http(
     let opt_clone = opt.clone();
     let index_scheduler = Data::from(index_scheduler);
     let auth_controller = Data::from(auth_controller);
+    let search_queue = SearchQueue::new(
+        opt.experimental_search_queue_size,
+        available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
+    );
+    let search_queue = Data::new(search_queue);
 
     let http_server = HttpServer::new(move || {
         create_app(
             index_scheduler.clone(),
             auth_controller.clone(),
+            search_queue.clone(),
             opt.clone(),
             logs.clone(),
             analytics.clone(),

meilisearch/src/middleware.rs

@@ -55,16 +55,17 @@ where
         let index_scheduler = req.app_data::<Data<IndexScheduler>>().unwrap();
         let features = index_scheduler.features();
 
+        let request_path = req.path();
+        let request_pattern = req.match_pattern();
+        let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str).to_string();
+        let request_method = req.method().to_string();
+
         if features.check_metrics().is_ok() {
-            let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
-                let request_pattern = req.match_pattern();
-                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
-                let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, metric_path])
+                        .with_label_values(&[&request_method, &metric_path])
                         .start_timer(),
                 );
             }
@@ -76,11 +77,7 @@ where
             let res = fut.await?;
 
             crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
-                .with_label_values(&[
-                    res.request().method().as_str(),
-                    res.request().path(),
-                    res.status().as_str(),
-                ])
+                .with_label_values(&[&request_method, &metric_path, res.status().as_str()])
                 .inc();
 
             if let Some(histogram_timer) = histogram_timer {

meilisearch/src/routes/indexes/facet_search.rs

@@ -81,7 +81,7 @@ pub async fn search(
     let index = index_scheduler.index(&index_uid)?;
     let features = index_scheduler.features();
     let search_kind = search_kind(&search_query, &index_scheduler, &index, features)?;
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_facet_search(
             &index,
@@ -93,7 +93,9 @@ pub async fn search(
             locales,
         )
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
 
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);

meilisearch/src/routes/indexes/search.rs

@@ -233,11 +233,13 @@ pub async fn search_with_url_query(
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)?;
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_search(&index, query, search_kind, retrieve_vector, index_scheduler.features())
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
@@ -276,11 +278,13 @@ pub async fn search_with_post(
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
 
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_search(&index, query, search_kind, retrieve_vectors, index_scheduler.features())
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {

meilisearch/src/routes/indexes/settings.rs

@@ -636,11 +636,19 @@ fn embedder_analytics(
             .any(|config| config.document_template.set().is_some())
     });
 
+    let document_template_max_bytes = setting.as_ref().and_then(|map| {
+        map.values()
+            .filter_map(|config| config.clone().set())
+            .filter_map(|config| config.document_template_max_bytes.set())
+            .max()
+    });
+
     json!(
         {
             "total": setting.as_ref().map(|s| s.len()),
             "sources": sources,
             "document_template_used": document_template_used,
+            "document_template_max_bytes": document_template_max_bytes
         }
     )
 }

meilisearch/src/routes/multi_search.rs

@@ -39,7 +39,7 @@ pub async fn multi_search_with_post(
 ) -> Result<HttpResponse, ResponseError> {
     // Since we don't want to process half of the search requests and then get a permit refused
     // we're going to get one permit for the whole duration of the multi-search request.
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
 
     let federated_search = params.into_inner();
 
@@ -81,6 +81,7 @@ pub async fn multi_search_with_post(
                 perform_federated_search(&index_scheduler, queries, federation, features)
             })
             .await;
+            permit.drop().await;
 
             if let Ok(Ok(_)) = search_result {
                 multi_aggregate.succeed();
@@ -143,6 +144,7 @@ pub async fn multi_search_with_post(
                 Ok(search_results)
             }
             .await;
+            permit.drop().await;
 
             if search_results.is_ok() {
                 multi_aggregate.succeed();

meilisearch/src/search_queue.rs

@@ -18,6 +18,7 @@
 //!                         And should drop the Permit only once you have freed all the RAM consumed by the method.
 
 use std::num::NonZeroUsize;
+use std::time::Duration;
 
 use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
@@ -29,16 +30,31 @@ use crate::error::MeilisearchHttpError;
 pub struct SearchQueue {
     sender: mpsc::Sender<oneshot::Sender<Permit>>,
     capacity: usize,
+    /// If we have waited longer than this to get a permit, we should abort the search request entirely.
+    /// The client probably already closed the connection, but we have no way to find out.
+    time_to_abort: Duration,
 }
 
 /// You should only run search requests while holding this permit.
 /// Once it's dropped, a new search request will be able to process.
+/// You should always try to drop the permit yourself calling the `drop` async method on it.
 #[derive(Debug)]
 pub struct Permit {
     sender: mpsc::Sender<()>,
 }
 
+impl Permit {
+    /// Drop the permit giving back on permit to the search queue.
+    pub async fn drop(self) {
+        // if the channel is closed then the whole instance is down
+        let _ = self.sender.send(()).await;
+    }
+}
+
 impl Drop for Permit {
+    /// The implicit drop implementation can still be called in multiple cases:
+    /// - We forgot to call the explicit one somewhere => this should be fixed on our side asap
+    /// - The future is cancelled while running and the permit dropped with it
     fn drop(&mut self) {
         let sender = self.sender.clone();
         // if the channel is closed then the whole instance is down
@@ -53,7 +69,11 @@ impl SearchQueue {
         let (sender, receiver) = mpsc::channel(1);
 
         tokio::task::spawn(Self::run(capacity, paralellism, receiver));
-        Self { sender, capacity }
+        Self { sender, capacity, time_to_abort: Duration::from_secs(60) }
+    }
+
+    pub fn with_time_to_abort(self, time_to_abort: Duration) -> Self {
+        Self { time_to_abort, ..self }
     }
 
     /// This function is the main loop, it's in charge on scheduling which search request should execute first and
@@ -119,9 +139,23 @@ impl SearchQueue {
     /// Returns a search `Permit`.
     /// It should be dropped as soon as you've freed all the RAM associated with the search request being processed.
     pub async fn try_get_search_permit(&self) -> Result<Permit, MeilisearchHttpError> {
+        let now = std::time::Instant::now();
         let (sender, receiver) = oneshot::channel();
         self.sender.send(sender).await.map_err(|_| MeilisearchHttpError::SearchLimiterIsDown)?;
-        receiver.await.map_err(|_| MeilisearchHttpError::TooManySearchRequests(self.capacity))
+        let permit = receiver
+            .await
+            .map_err(|_| MeilisearchHttpError::TooManySearchRequests(self.capacity))?;
+
+        // If we've been for more than one minute to get a search permit, it's better to simply
+        // abort the search request than spending time processing something were the client
+        // most certainly exited or got a timeout a long time ago.
+        // We may find a better solution in https://github.com/actix/actix-web/issues/3462.
+        if now.elapsed() > self.time_to_abort {
+            permit.drop().await;
+            Err(MeilisearchHttpError::TooManySearchRequests(self.capacity))
+        } else {
+            Ok(permit)
+        }
     }
 
     /// Returns `Ok(())` if everything seems normal.

meilisearch/tests/auth/mod.rs

@@ -5,63 +5,3 @@ mod payload;
 mod tenant_token;
 
 mod tenant_token_multi_search;
-
-use actix_web::http::StatusCode;
-
-use crate::common::{Server, Value};
-use crate::json;
-
-impl Server {
-    pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {
-        self.service.api_key = Some(api_key.as_ref().to_string());
-    }
-
-    /// Fetch and use the default admin key for nexts http requests.
-    pub async fn use_admin_key(&mut self, master_key: impl AsRef<str>) {
-        self.use_api_key(master_key);
-        let (response, code) = self.list_api_keys("").await;
-        assert_eq!(200, code, "{:?}", response);
-        let admin_key = &response["results"][1]["key"];
-        self.use_api_key(admin_key.as_str().unwrap());
-    }
-
-    pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) {
-        let url = "/keys";
-        self.service.post(url, content).await
-    }
-
-    pub async fn get_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.get(url).await
-    }
-
-    pub async fn patch_api_key(&self, key: impl AsRef<str>, content: Value) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.patch(url, content).await
-    }
-
-    pub async fn list_api_keys(&self, params: &str) -> (Value, StatusCode) {
-        let url = format!("/keys{params}");
-        self.service.get(url).await
-    }
-
-    pub async fn delete_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.delete(url).await
-    }
-
-    pub async fn dummy_request(
-        &self,
-        method: impl AsRef<str>,
-        url: impl AsRef<str>,
-    ) -> (Value, StatusCode) {
-        match method.as_ref() {
-            "POST" => self.service.post(url, json!({})).await,
-            "PUT" => self.service.put(url, json!({})).await,
-            "PATCH" => self.service.patch(url, json!({})).await,
-            "GET" => self.service.get(url).await,
-            "DELETE" => self.service.delete(url).await,
-            _ => unreachable!(),
-        }
-    }
-}

meilisearch/tests/auth/tenant_token.rs

@@ -6,7 +6,7 @@ use once_cell::sync::Lazy;
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::{Server, Value};
+use crate::common::{Server, Value, DOCUMENTS};
 use crate::json;
 
 fn generate_tenant_token(
@@ -22,36 +22,6 @@ fn generate_tenant_token(
         .unwrap()
 }
 
-static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Shazam!",
-            "id": "287947",
-            "color": ["green", "blue"]
-        },
-        {
-            "title": "Captain Marvel",
-            "id": "299537",
-            "color": ["yellow", "blue"]
-        },
-        {
-            "title": "Escape Room",
-            "id": "522681",
-            "color": ["yellow", "red"]
-        },
-        {
-            "title": "How to Train Your Dragon: The Hidden World",
-            "id": "166428",
-            "color": ["green", "red"]
-        },
-        {
-            "title": "Glass",
-            "id": "450465",
-            "color": ["blue", "red"]
-        }
-    ])
-});
-
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
     json!({
         "message": null,

meilisearch/tests/common/index.rs

@@ -1,4 +1,5 @@
 use std::fmt::Write;
+use std::marker::PhantomData;
 use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
@@ -9,19 +10,24 @@ use urlencoding::encode as urlencode;
 use super::encoder::Encoder;
 use super::service::Service;
 use super::Value;
+use super::{Owned, Shared};
 use crate::json;
 
-pub struct Index<'a> {
+pub struct Index<'a, State = Owned> {
     pub uid: String,
     pub service: &'a Service,
-    pub encoder: Encoder,
+    pub(super) encoder: Encoder,
+    pub(super) marker: PhantomData<State>,
 }
 
-#[allow(dead_code)]
-impl Index<'_> {
-    pub async fn get(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
-        self.service.get(url).await
+impl<'a> Index<'a, Owned> {
+    pub fn to_shared(&self) -> Index<'a, Shared> {
+        Index {
+            uid: self.uid.clone(),
+            service: self.service,
+            encoder: self.encoder,
+            marker: PhantomData,
+        }
     }
 
     pub async fn load_test_set(&self) -> u64 {
@@ -57,11 +63,7 @@ impl Index<'_> {
     }
 
     pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
-        let body = json!({
-            "uid": self.uid,
-            "primaryKey": primary_key,
-        });
-        self.service.post_encoded("/indexes", body, self.encoder).await
+        self._create(primary_key).await
     }
 
     pub async fn update_raw(&self, body: Value) -> (Value, StatusCode) {
@@ -88,13 +90,7 @@ impl Index<'_> {
         documents: Value,
         primary_key: Option<&str>,
     ) -> (Value, StatusCode) {
-        let url = match primary_key {
-            Some(key) => {
-                format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
-            }
-            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
-        };
-        self.service.post_encoded(url, documents, self.encoder).await
+        self._add_documents(documents, primary_key).await
     }
 
     pub async fn raw_add_documents(
@@ -136,80 +132,11 @@ impl Index<'_> {
         }
     }
 
-    pub async fn wait_task(&self, update_id: u64) -> Value {
-        // try several times to get status, or panic to not wait forever
-        let url = format!("/tasks/{}", update_id);
-        for _ in 0..100 {
-            let (response, status_code) = self.service.get(&url).await;
-            assert_eq!(200, status_code, "response: {}", response);
-
-            if response["status"] == "succeeded" || response["status"] == "failed" {
-                return response;
-            }
-
-            // wait 0.5 second.
-            sleep(Duration::from_millis(500)).await;
-        }
-        panic!("Timeout waiting for update id");
-    }
-
-    pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
-        let url = format!("/tasks/{}", update_id);
-        self.service.get(url).await
-    }
-
     pub async fn list_tasks(&self) -> (Value, StatusCode) {
         let url = format!("/tasks?indexUids={}", self.uid);
         self.service.get(url).await
     }
 
-    pub async fn filtered_tasks(
-        &self,
-        types: &[&str],
-        statuses: &[&str],
-        canceled_by: &[&str],
-    ) -> (Value, StatusCode) {
-        let mut url = format!("/tasks?indexUids={}", self.uid);
-        if !types.is_empty() {
-            let _ = write!(url, "&types={}", types.join(","));
-        }
-        if !statuses.is_empty() {
-            let _ = write!(url, "&statuses={}", statuses.join(","));
-        }
-        if !canceled_by.is_empty() {
-            let _ = write!(url, "&canceledBy={}", canceled_by.join(","));
-        }
-        self.service.get(url).await
-    }
-
-    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
-        if let Some(options) = options {
-            write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
-        }
-        self.service.get(url).await
-    }
-
-    pub async fn get_document_by_filter(&self, payload: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/fetch", urlencode(self.uid.as_ref()));
-        self.service.post(url, payload).await
-    }
-
-    pub async fn get_all_documents_raw(&self, options: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), options);
-        self.service.get(url).await
-    }
-
-    pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let url = format!(
-            "/indexes/{}/documents{}",
-            urlencode(self.uid.as_ref()),
-            yaup::to_string(&options).unwrap()
-        );
-
-        self.service.get(url).await
-    }
-
     pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         self.service.delete(url).await
@@ -237,14 +164,8 @@ impl Index<'_> {
         self.service.post_encoded(url, body, self.encoder).await
     }
 
-    pub async fn settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.get(url).await
-    }
-
     pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.patch_encoded(url, settings, self.encoder).await
+        self._update_settings(settings).await
     }
 
     pub async fn update_settings_displayed_attributes(
@@ -327,6 +248,146 @@ impl Index<'_> {
         self.service.delete(url).await
     }
 
+    pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
+        let url =
+            format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
+        self.service.put_encoded(url, value, self.encoder).await
+    }
+}
+
+impl<'a> Index<'a, Shared> {
+    /// You cannot modify the content of a shared index, thus the delete_document_by_filter call
+    /// must fail. If the task successfully enqueue itself, we'll wait for the task to finishes,
+    /// and if it succeed the function will panic.
+    pub async fn delete_document_by_filter_fail(&self, body: Value) -> (Value, StatusCode) {
+        let (mut task, code) = self._delete_document_by_filter(body).await;
+        if code.is_success() {
+            task = self.wait_task(task.uid()).await;
+            if task.is_success() {
+                panic!(
+                    "`delete_document_by_filter_fail` succeeded: {}",
+                    serde_json::to_string_pretty(&task).unwrap()
+                );
+            }
+        }
+        (task, code)
+    }
+}
+
+#[allow(dead_code)]
+impl<State> Index<'_, State> {
+    pub async fn get(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
+        self.service.get(url).await
+    }
+
+    /// add_documents is not allowed on shared index but we need to use it to initialize
+    /// a bunch of very common indexes in `common/mod.rs`.
+    pub(super) async fn _add_documents(
+        &self,
+        documents: Value,
+        primary_key: Option<&str>,
+    ) -> (Value, StatusCode) {
+        let url = match primary_key {
+            Some(key) => {
+                format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
+            }
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
+        };
+        self.service.post_encoded(url, documents, self.encoder).await
+    }
+
+    pub(super) async fn _update_settings(&self, settings: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
+        self.service.patch_encoded(url, settings, self.encoder).await
+    }
+
+    pub(super) async fn _delete_document_by_filter(&self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/delete", urlencode(self.uid.as_ref()));
+        self.service.post_encoded(url, body, self.encoder).await
+    }
+
+    pub(super) async fn _create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
+        let body = json!({
+            "uid": self.uid,
+            "primaryKey": primary_key,
+        });
+        self.service.post_encoded("/indexes", body, self.encoder).await
+    }
+    pub async fn wait_task(&self, update_id: u64) -> Value {
+        // try several times to get status, or panic to not wait forever
+        let url = format!("/tasks/{}", update_id);
+        for _ in 0..100 {
+            let (response, status_code) = self.service.get(&url).await;
+            assert_eq!(200, status_code, "response: {}", response);
+
+            if response["status"] == "succeeded" || response["status"] == "failed" {
+                return response;
+            }
+
+            // wait 0.5 second.
+            sleep(Duration::from_millis(500)).await;
+        }
+        panic!("Timeout waiting for update id");
+    }
+
+    pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
+        let url = format!("/tasks/{}", update_id);
+        self.service.get(url).await
+    }
+
+    pub async fn filtered_tasks(
+        &self,
+        types: &[&str],
+        statuses: &[&str],
+        canceled_by: &[&str],
+    ) -> (Value, StatusCode) {
+        let mut url = format!("/tasks?indexUids={}", self.uid);
+        if !types.is_empty() {
+            let _ = write!(url, "&types={}", types.join(","));
+        }
+        if !statuses.is_empty() {
+            let _ = write!(url, "&statuses={}", statuses.join(","));
+        }
+        if !canceled_by.is_empty() {
+            let _ = write!(url, "&canceledBy={}", canceled_by.join(","));
+        }
+        self.service.get(url).await
+    }
+
+    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
+        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
+        if let Some(options) = options {
+            write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
+        }
+        self.service.get(url).await
+    }
+
+    pub async fn get_document_by_filter(&self, payload: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/fetch", urlencode(self.uid.as_ref()));
+        self.service.post(url, payload).await
+    }
+
+    pub async fn get_all_documents_raw(&self, options: &str) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), options);
+        self.service.get(url).await
+    }
+
+    pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
+        let url = format!(
+            "/indexes/{}/documents{}",
+            urlencode(self.uid.as_ref()),
+            yaup::to_string(&options).unwrap()
+        );
+
+        self.service.get(url).await
+    }
+
+    pub async fn settings(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
+        self.service.get(url).await
+    }
+
     pub async fn stats(&self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
         self.service.get(url).await
@@ -411,12 +472,6 @@ impl Index<'_> {
         self.service.post_encoded(url, query, self.encoder).await
     }
 
-    pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
-        let url =
-            format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
-        self.service.put_encoded(url, value, self.encoder).await
-    }
-
     pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
         let url =
             format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");

meilisearch/tests/common/mod.rs

@@ -8,9 +8,16 @@ use std::fmt::{self, Display};
 #[allow(unused)]
 pub use index::GetAllDocumentsOptions;
 use meili_snap::json_string;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 #[allow(unused)]
 pub use server::{default_settings, Server};
+use tokio::sync::OnceCell;
+
+use crate::common::index::Index;
+
+pub enum Shared {}
+pub enum Owned {}
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct Value(pub serde_json::Value);
@@ -27,10 +34,20 @@ impl Value {
         }
     }
 
+    /// Return `true` if the `status` field is set to `succeeded`.
+    /// Panic if the `status` field doesn't exists.
+    #[track_caller]
+    pub fn is_success(&self) -> bool {
+        if !self["status"].is_string() {
+            panic!("Called `is_success` on {}", serde_json::to_string_pretty(&self.0).unwrap());
+        }
+        self["status"] == serde_json::Value::String(String::from("succeeded"))
+    }
+
     // Panic if the json doesn't contain the `status` field set to "succeeded"
     #[track_caller]
     pub fn succeeded(&self) -> &Self {
-        if self["status"] != serde_json::Value::String(String::from("succeeded")) {
+        if !self.is_success() {
             panic!("Called succeeded on {}", serde_json::to_string_pretty(&self.0).unwrap());
         }
         self
@@ -122,3 +139,253 @@ macro_rules! test_post_get_search {
             .map_err(|e| panic!("panic in post route: {:?}", e.downcast_ref::<&str>().unwrap()));
     };
 }
+
+pub async fn shared_does_not_exists_index() -> &'static Index<'static, Shared> {
+    static INDEX: Lazy<Index<'static, Shared>> = Lazy::new(|| {
+        let server = Server::new_shared();
+        server._index("DOES_NOT_EXISTS").to_shared()
+    });
+    &INDEX
+}
+
+pub async fn shared_empty_index() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+
+    INDEX
+        .get_or_init(|| async {
+            let server = Server::new_shared();
+            let index = server._index("EMPTY_INDEX").to_shared();
+            let (response, _code) = index._create(None).await;
+            index.wait_task(response.uid()).await.succeeded();
+            index
+        })
+        .await
+}
+
+pub static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "title": "Shazam!",
+            "id": "287947",
+            "color": ["green", "blue"],
+            "_vectors": { "manual": [1, 2, 3]},
+        },
+        {
+            "title": "Captain Marvel",
+            "id": "299537",
+            "color": ["yellow", "blue"],
+            "_vectors": { "manual": [1, 2, 54] },
+        },
+        {
+            "title": "Escape Room",
+            "id": "522681",
+            "color": ["yellow", "red"],
+            "_vectors": { "manual": [10, -23, 32] },
+        },
+        {
+            "title": "How to Train Your Dragon: The Hidden World",
+            "id": "166428",
+            "color": ["green", "red"],
+            "_vectors": { "manual": [-100, 231, 32] },
+        },
+        {
+            "title": "Gläss",
+            "id": "450465",
+            "color": ["blue", "red"],
+            "_vectors": { "manual": [-100, 340, 90] },
+        }
+    ])
+});
+
+pub async fn shared_index_with_documents() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+    INDEX.get_or_init(|| async {
+        let server = Server::new_shared();
+        let index = server._index("SHARED_DOCUMENTS").to_shared();
+        let documents = DOCUMENTS.clone();
+        let (response, _code) = index._add_documents(documents, None).await;
+        index.wait_task(response.uid()).await.succeeded();
+        let (response, _code) = index
+            ._update_settings(
+                json!({"filterableAttributes": ["id", "title"], "sortableAttributes": ["id", "title"]}),
+            )
+            .await;
+        index.wait_task(response.uid()).await.succeeded();
+        index
+    }).await
+}
+
+pub static SCORE_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "title": "Batman the dark knight returns: Part 1",
+            "id": "A",
+        },
+        {
+            "title": "Batman the dark knight returns: Part 2",
+            "id": "B",
+        },
+        {
+            "title": "Batman Returns",
+            "id": "C",
+        },
+        {
+            "title": "Batman",
+            "id": "D",
+        },
+        {
+            "title": "Badman",
+            "id": "E",
+        }
+    ])
+});
+
+pub static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "id": 852,
+            "father": "jean",
+            "mother": "michelle",
+            "doggos": [
+                {
+                    "name": "bobby",
+                    "age": 2,
+                },
+                {
+                    "name": "buddy",
+                    "age": 4,
+                },
+            ],
+            "cattos": "pésti",
+            "_vectors": { "manual": [1, 2, 3]},
+        },
+        {
+            "id": 654,
+            "father": "pierre",
+            "mother": "sabine",
+            "doggos": [
+                {
+                    "name": "gros bill",
+                    "age": 8,
+                },
+            ],
+            "cattos": ["simba", "pestiféré"],
+            "_vectors": { "manual": [1, 2, 54] },
+        },
+        {
+            "id": 750,
+            "father": "romain",
+            "mother": "michelle",
+            "cattos": ["enigma"],
+            "_vectors": { "manual": [10, 23, 32] },
+        },
+        {
+            "id": 951,
+            "father": "jean-baptiste",
+            "mother": "sophie",
+            "doggos": [
+                {
+                    "name": "turbo",
+                    "age": 5,
+                },
+                {
+                    "name": "fast",
+                    "age": 6,
+                },
+            ],
+            "cattos": ["moumoute", "gomez"],
+            "_vectors": { "manual": [10, 23, 32] },
+        },
+    ])
+});
+
+pub async fn shared_index_with_nested_documents() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+    INDEX.get_or_init(|| async {
+        let server = Server::new_shared();
+        let index = server._index("SHARED_NESTED_DOCUMENTS").to_shared();
+        let documents = NESTED_DOCUMENTS.clone();
+        let (response, _code) = index._add_documents(documents, None).await;
+        index.wait_task(response.uid()).await.succeeded();
+        let (response, _code) = index
+            ._update_settings(
+                json!({"filterableAttributes": ["father", "doggos"], "sortableAttributes": ["doggos"]}),
+            )
+            .await;
+        index.wait_task(response.uid()).await.succeeded();
+        index
+    }).await
+}
+
+pub static FRUITS_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "name": "Exclusive sale: green apple",
+            "id": "green-apple-boosted",
+            "BOOST": true
+        },
+        {
+            "name": "Pear",
+            "id": "pear",
+        },
+        {
+            "name": "Red apple gala",
+            "id": "red-apple-gala",
+        },
+        {
+            "name": "Exclusive sale: Red Tomato",
+            "id": "red-tomatoes-boosted",
+            "BOOST": true
+        },
+        {
+            "name": "Exclusive sale: Red delicious apple",
+            "id": "red-delicious-boosted",
+            "BOOST": true,
+        }
+    ])
+});
+
+pub static VECTOR_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+      {
+        "id": "A",
+        "description": "the dog barks at the cat",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.9, 0.8, 0.05],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-0.1, 0.55]
+        }
+      },
+      {
+        "id": "B",
+        "description": "the kitten scratched the beagle",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.8, 0.9, 0.5],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-0.2, 0.65]
+        }
+      },
+      {
+        "id": "C",
+        "description": "the dog had to stay alone today",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.85, 0.02, 0.1],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-1.0, 0.1]
+        }
+      },
+      {
+        "id": "D",
+        "description": "the little boy pets the puppy",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.8, 0.09, 0.8],
+          // dimensions [negative/positive, energy]
+          "sentiment": [0.8, 0.3]
+        }
+      },
+    ])
+});

meilisearch/tests/common/server.rs

@@ -1,7 +1,8 @@
 #![allow(dead_code)]
 
+use std::marker::PhantomData;
 use std::path::Path;
-use std::str::FromStr as _;
+use std::str::FromStr;
 use std::time::Duration;
 
 use actix_http::body::MessageBody;
@@ -10,28 +11,34 @@ use actix_web::http::StatusCode;
 use byte_unit::{Byte, Unit};
 use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, MaxThreads, Opt};
-use meilisearch::{analytics, create_app, setup_meilisearch, SubscriberForSecondLayer};
+use meilisearch::setup_meilisearch;
 use once_cell::sync::Lazy;
 use tempfile::TempDir;
+use tokio::sync::OnceCell;
 use tokio::time::sleep;
-use tracing::level_filters::LevelFilter;
-use tracing_subscriber::Layer;
+use uuid::Uuid;
 
 use super::index::Index;
 use super::service::Service;
+use super::{Owned, Shared};
 use crate::common::encoder::Encoder;
 use crate::common::Value;
 use crate::json;
 
-pub struct Server {
+pub struct Server<State = Owned> {
     pub service: Service,
     // hold ownership to the tempdir while we use the server instance.
     _dir: Option<TempDir>,
+    _marker: PhantomData<State>,
 }
 
 pub static TEST_TEMP_DIR: Lazy<TempDir> = Lazy::new(|| TempDir::new().unwrap());
 
-impl Server {
+impl Server<Owned> {
+    fn into_shared(self) -> Server<Shared> {
+        Server { service: self.service, _dir: self._dir, _marker: PhantomData }
+    }
+
     pub async fn new() -> Self {
         let dir = TempDir::new().unwrap();
 
@@ -46,7 +53,7 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Server { service, _dir: Some(dir) }
+        Server { service, _dir: Some(dir), _marker: PhantomData }
     }
 
     pub async fn new_auth_with_options(mut options: Opt, dir: TempDir) -> Self {
@@ -61,7 +68,7 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Server { service, _dir: Some(dir) }
+        Server { service, _dir: Some(dir), _marker: PhantomData }
     }
 
     pub async fn new_auth() -> Self {
@@ -74,38 +81,35 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options)?;
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Ok(Server { service, _dir: None })
+        Ok(Server { service, _dir: None, _marker: PhantomData })
     }
 
-    pub async fn init_web_app(
-        &self,
-    ) -> impl actix_web::dev::Service<
-        actix_http::Request,
-        Response = ServiceResponse<impl MessageBody>,
-        Error = actix_web::Error,
-    > {
-        let (_route_layer, route_layer_handle) =
-            tracing_subscriber::reload::Layer::new(None.with_filter(
-                tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
-            ));
-        let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
-            (Box::new(
-                tracing_subscriber::fmt::layer()
-                    .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
-            )
-                as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
-                .with_filter(tracing_subscriber::filter::Targets::new()),
-        );
+    pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {
+        self.service.api_key = Some(api_key.as_ref().to_string());
+    }
 
-        actix_web::test::init_service(create_app(
-            self.service.index_scheduler.clone().into(),
-            self.service.auth.clone().into(),
-            self.service.options.clone(),
-            (route_layer_handle, stderr_layer_handle),
-            analytics::MockAnalytics::new(&self.service.options),
-            true,
-        ))
-        .await
+    /// Fetch and use the default admin key for nexts http requests.
+    pub async fn use_admin_key(&mut self, master_key: impl AsRef<str>) {
+        self.use_api_key(master_key);
+        let (response, code) = self.list_api_keys("").await;
+        assert_eq!(200, code, "{:?}", response);
+        let admin_key = &response["results"][1]["key"];
+        self.use_api_key(admin_key.as_str().unwrap());
+    }
+
+    pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) {
+        let url = "/keys";
+        self.service.post(url, content).await
+    }
+
+    pub async fn patch_api_key(&self, key: impl AsRef<str>, content: Value) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.patch(url, content).await
+    }
+
+    pub async fn delete_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.delete(url).await
     }
 
     /// Returns a view to an index. There is no guarantee that the index exists.
@@ -118,15 +122,12 @@ impl Server {
     }
 
     pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
-        Index { uid: uid.as_ref().to_string(), service: &self.service, encoder }
-    }
-
-    pub async fn multi_search(&self, queries: Value) -> (Value, StatusCode) {
-        self.service.post("/multi-search", queries).await
-    }
-
-    pub async fn list_indexes_raw(&self, parameters: &str) -> (Value, StatusCode) {
-        self.service.get(format!("/indexes{parameters}")).await
+        Index {
+            uid: uid.as_ref().to_string(),
+            service: &self.service,
+            encoder,
+            marker: PhantomData,
+        }
     }
 
     pub async fn list_indexes(
@@ -150,10 +151,6 @@ impl Server {
         }
     }
 
-    pub async fn version(&self) -> (Value, StatusCode) {
-        self.service.get("/version").await
-    }
-
     pub async fn stats(&self) -> (Value, StatusCode) {
         self.service.get("/stats").await
     }
@@ -162,12 +159,174 @@ impl Server {
         self.service.get("/tasks").await
     }
 
+    pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
+        self.service.patch("/experimental-features", value).await
+    }
+
+    pub async fn get_metrics(&self) -> (Value, StatusCode) {
+        self.service.get("/metrics").await
+    }
+}
+
+impl Server<Shared> {
+    fn init_new_shared_instance() -> Server<Shared> {
+        let dir = TempDir::new().unwrap();
+
+        if cfg!(windows) {
+            std::env::set_var("TMP", TEST_TEMP_DIR.path());
+        } else {
+            std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
+        }
+
+        let options = default_settings(dir.path());
+
+        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
+        let service = Service { index_scheduler, auth, api_key: None, options };
+
+        Server { service, _dir: Some(dir), _marker: PhantomData }
+    }
+
+    pub fn new_shared() -> &'static Server<Shared> {
+        static SERVER: Lazy<Server<Shared>> = Lazy::new(Server::init_new_shared_instance);
+        &SERVER
+    }
+
+    pub async fn new_shared_with_admin_key() -> &'static Server<Shared> {
+        static SERVER: OnceCell<Server<Shared>> = OnceCell::const_new();
+        SERVER
+            .get_or_init(|| async {
+                let mut server = Server::new_auth().await;
+                server.use_admin_key("MASTER_KEY").await;
+                server.into_shared()
+            })
+            .await
+    }
+
+    /// You shouldn't access random indexes on a shared instance thus this method
+    /// must fail.
+    pub async fn get_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.get(url).await;
+        if code.is_success() {
+            panic!("`get_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    pub async fn delete_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.delete(url).await;
+        if code.is_success() {
+            panic!("`delete_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    pub async fn update_raw_index_fail(
+        &self,
+        uid: impl AsRef<str>,
+        body: Value,
+    ) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.patch_encoded(url, body, Encoder::Plain).await;
+        if code.is_success() {
+            panic!("`update_raw_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    /// Since this call updates the state of the instance, it must fail.
+    /// If it doesn't fail, the test will panic to help you debug what
+    /// is going on.
+    pub async fn create_index_fail(&self, body: Value) -> (Value, StatusCode) {
+        let (mut task, code) = self._create_index(body).await;
+        if code.is_success() {
+            task = self.wait_task(task.uid()).await;
+            if task.is_success() {
+                panic!(
+                    "`create_index_fail` succeeded: {}",
+                    serde_json::to_string_pretty(&task).unwrap()
+                );
+            }
+        }
+        (task, code)
+    }
+}
+
+impl<State> Server<State> {
+    pub async fn init_web_app(
+        &self,
+    ) -> impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = ServiceResponse<impl MessageBody>,
+        Error = actix_web::Error,
+    > {
+        self.service.init_web_app().await
+    }
+
+    pub async fn list_api_keys(&self, params: &str) -> (Value, StatusCode) {
+        let url = format!("/keys{params}");
+        self.service.get(url).await
+    }
+
+    pub async fn dummy_request(
+        &self,
+        method: impl AsRef<str>,
+        url: impl AsRef<str>,
+    ) -> (Value, StatusCode) {
+        match method.as_ref() {
+            "POST" => self.service.post(url, json!({})).await,
+            "PUT" => self.service.put(url, json!({})).await,
+            "PATCH" => self.service.patch(url, json!({})).await,
+            "GET" => self.service.get(url).await,
+            "DELETE" => self.service.delete(url).await,
+            _ => unreachable!(),
+        }
+    }
+
+    pub async fn get_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.get(url).await
+    }
+
+    pub(super) fn _index(&self, uid: impl AsRef<str>) -> Index<'_> {
+        Index {
+            uid: uid.as_ref().to_string(),
+            service: &self.service,
+            encoder: Encoder::Plain,
+            marker: PhantomData,
+        }
+    }
+
+    /// Returns a view to an index. There is no guarantee that the index exists.
+    pub fn unique_index(&self) -> Index<'_> {
+        let uuid = Uuid::new_v4();
+        Index {
+            uid: uuid.to_string(),
+            service: &self.service,
+            encoder: Encoder::Plain,
+            marker: PhantomData,
+        }
+    }
+
+    pub(super) async fn _create_index(&self, body: Value) -> (Value, StatusCode) {
+        self.service.post("/indexes", body).await
+    }
+
+    pub async fn multi_search(&self, queries: Value) -> (Value, StatusCode) {
+        self.service.post("/multi-search", queries).await
+    }
+
+    pub async fn list_indexes_raw(&self, parameters: &str) -> (Value, StatusCode) {
+        self.service.get(format!("/indexes{parameters}")).await
+    }
+
     pub async fn tasks_filter(&self, filter: &str) -> (Value, StatusCode) {
         self.service.get(format!("/tasks?{}", filter)).await
     }
 
-    pub async fn get_dump_status(&self, uid: &str) -> (Value, StatusCode) {
-        self.service.get(format!("/dumps/{}/status", uid)).await
+    pub async fn version(&self) -> (Value, StatusCode) {
+        self.service.get("/version").await
     }
 
     pub async fn create_dump(&self) -> (Value, StatusCode) {
@@ -215,14 +374,6 @@ impl Server {
     pub async fn get_features(&self) -> (Value, StatusCode) {
         self.service.get("/experimental-features").await
     }
-
-    pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
-        self.service.patch("/experimental-features", value).await
-    }
-
-    pub async fn get_metrics(&self) -> (Value, StatusCode) {
-        self.service.get("/metrics").await
-    }
 }
 
 pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
@@ -240,7 +391,8 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
             // memory has to be unlimited because several meilisearch are running in test context.
             max_indexing_memory: MaxMemory::unlimited(),
             skip_index_budget: true,
-            max_indexing_threads: MaxThreads::from_str("1").unwrap(),
+            // Having 2 threads makes the tests way faster
+            max_indexing_threads: MaxThreads::from_str("2").unwrap(),
         },
         experimental_enable_metrics: false,
         ..Parser::parse_from(None as Option<&str>)

meilisearch/tests/common/service.rs

@@ -1,10 +1,15 @@
+use std::num::NonZeroUsize;
 use std::sync::Arc;
 
+use actix_web::body::MessageBody;
+use actix_web::dev::ServiceResponse;
 use actix_web::http::header::ContentType;
 use actix_web::http::StatusCode;
 use actix_web::test;
 use actix_web::test::TestRequest;
+use actix_web::web::Data;
 use index_scheduler::IndexScheduler;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
 use meilisearch_auth::AuthController;
 use tracing::level_filters::LevelFilter;
@@ -106,7 +111,13 @@ impl Service {
         self.request(req).await
     }
 
-    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
+    pub async fn init_web_app(
+        &self,
+    ) -> impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = ServiceResponse<impl MessageBody>,
+        Error = actix_web::Error,
+    > {
         let (_route_layer, route_layer_handle) =
             tracing_subscriber::reload::Layer::new(None.with_filter(
                 tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
@@ -119,16 +130,25 @@ impl Service {
                 as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
                 .with_filter(tracing_subscriber::filter::Targets::new()),
         );
+        let search_queue = SearchQueue::new(
+            self.options.experimental_search_queue_size,
+            NonZeroUsize::new(1).unwrap(),
+        );
 
-        let app = test::init_service(create_app(
+        actix_web::test::init_service(create_app(
             self.index_scheduler.clone().into(),
             self.auth.clone().into(),
+            Data::new(search_queue),
             self.options.clone(),
             (route_layer_handle, stderr_layer_handle),
             analytics::MockAnalytics::new(&self.options),
             true,
         ))
-        .await;
+        .await
+    }
+
+    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
+        let app = self.init_web_app().await;
 
         if let Some(api_key) = &self.api_key {
             req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));

meilisearch/tests/content_type.rs

@@ -6,6 +6,7 @@ use actix_web::test;
 
 use crate::common::{Server, Value};
 
+#[derive(Debug)]
 enum HttpVerb {
     Put,
     Patch,
@@ -80,7 +81,7 @@ async fn error_json_bad_content_type() {
         let status_code = res.status();
         let body = test::read_body(res).await;
         let response: Value = serde_json::from_slice(&body).unwrap_or_default();
-        assert_eq!(status_code, 415, "calling the route `{}` without content-type is supposed to throw a bad media type error", route);
+        assert_eq!(status_code, 415, "calling the route `{verb:?} {route}` without content-type is supposed to throw a bad media type error:\n{}", String::from_utf8_lossy(&body));
         assert_eq!(
             response,
             json!({

meilisearch/tests/documents/errors.rs

@@ -1,14 +1,15 @@
 use meili_snap::*;
 use urlencoding::encode;
 
-use crate::common::Server;
+use crate::common::{
+    shared_does_not_exists_index, shared_empty_index, shared_index_with_documents, Server,
+};
 use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.get_all_documents_raw("?offset").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -45,9 +46,8 @@ async fn get_all_documents_bad_offset() {
 
 #[actix_rt::test]
 async fn get_all_documents_bad_limit() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.get_all_documents_raw("?limit").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -84,17 +84,13 @@ async fn get_all_documents_bad_limit() {
 
 #[actix_rt::test]
 async fn get_all_documents_bad_filter() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let index = shared_does_not_exists_index().await;
 
-    // Since the filter can't be parsed automatically by deserr, we have the wrong error message
-    // if the index does not exist: we could expect to get an error message about the invalid filter before
-    // the existence of the index is checked, but it is not the case.
     let (response, code) = index.get_all_documents_raw("?filter").await;
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -105,7 +101,7 @@ async fn get_all_documents_bad_filter() {
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -116,42 +112,14 @@ async fn get_all_documents_bad_filter() {
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
     }
     "###);
 
-    let (response, code) = index.create(None).await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 0,
-      "indexUid": "test",
-      "status": "enqueued",
-      "type": "indexCreation",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let response = server.wait_task(0).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }), @r###"
-    {
-      "uid": 0,
-      "indexUid": "test",
-      "status": "succeeded",
-      "type": "indexCreation",
-      "canceledBy": null,
-      "details": {
-        "primaryKey": null
-      },
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
+    let index = shared_empty_index().await;
 
     let (response, code) = index.get_all_documents_raw("?filter").await;
     snapshot!(code, @"200 OK");
@@ -189,9 +157,8 @@ async fn get_all_documents_bad_filter() {
 
 #[actix_rt::test]
 async fn delete_documents_batch() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.delete_batch_raw(json!("doggo")).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -206,9 +173,8 @@ async fn delete_documents_batch() {
 
 #[actix_rt::test]
 async fn replace_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_add_documents("", vec![("Content-Type", "application/json")], "").await;
     snapshot!(code, @"400 Bad Request");
@@ -248,9 +214,8 @@ async fn replace_documents_missing_payload() {
 
 #[actix_rt::test]
 async fn update_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", Some("application/json"), "").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -287,9 +252,8 @@ async fn update_documents_missing_payload() {
 
 #[actix_rt::test]
 async fn replace_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_add_documents("", Vec::new(), "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -316,9 +280,8 @@ async fn replace_documents_missing_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", None, "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -345,9 +308,8 @@ async fn update_documents_missing_content_type() {
 
 #[actix_rt::test]
 async fn replace_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_add_documents("", vec![("Content-Type", "doggo")], "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -362,9 +324,8 @@ async fn replace_documents_bad_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", Some("doggo"), "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -379,9 +340,8 @@ async fn update_documents_bad_content_type() {
 
 #[actix_rt::test]
 async fn replace_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index
         .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter")
         .await;
@@ -428,9 +388,8 @@ async fn replace_documents_bad_csv_delimiter() {
 
 #[actix_rt::test]
 async fn update_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_update_documents("", Some("application/json"), "?csvDelimiter").await;
     snapshot!(code, @"400 Bad Request");
@@ -475,9 +434,8 @@ async fn update_documents_bad_csv_delimiter() {
 
 #[actix_rt::test]
 async fn replace_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index
         .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter=a")
         .await;
@@ -507,9 +465,8 @@ async fn replace_documents_csv_delimiter_with_bad_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_update_documents("", Some("application/json"), "?csvDelimiter=a").await;
     snapshot!(code, @"415 Unsupported Media Type");
@@ -537,13 +494,12 @@ async fn update_documents_csv_delimiter_with_bad_content_type() {
 
 #[actix_rt::test]
 async fn delete_document_by_filter() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    // send a bad payload type
     let (response, code) = index.delete_document_by_filter(json!("hello")).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type: expected an object, but found a string: `\"hello\"`",
       "code": "bad_request",
@@ -555,7 +511,7 @@ async fn delete_document_by_filter() {
     // send bad payload type
     let (response, code) = index.delete_document_by_filter(json!({ "filter": true })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid syntax for the filter parameter: `expected String, Array, found: true`.",
       "code": "invalid_document_filter",
@@ -567,7 +523,7 @@ async fn delete_document_by_filter() {
     // send bad filter
     let (response, code) = index.delete_document_by_filter(json!({ "filter": "hello"})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `hello`.\n1:6 hello",
       "code": "invalid_document_filter",
@@ -579,7 +535,7 @@ async fn delete_document_by_filter() {
     // send empty filter
     let (response, code) = index.delete_document_by_filter(json!({ "filter": ""})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Sending an empty filter is forbidden.",
       "code": "invalid_document_filter",
@@ -591,7 +547,7 @@ async fn delete_document_by_filter() {
     // do not send any filter
     let (response, code) = index.delete_document_by_filter(json!({})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Missing field `filter`",
       "code": "missing_document_filter",
@@ -600,15 +556,14 @@ async fn delete_document_by_filter() {
     }
     "###);
 
+    let index = shared_does_not_exists_index().await;
     // index does not exists
-    let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "doggo = bernese"})).await;
-    snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let (response, _code) =
+        index.delete_document_by_filter_fail(json!({ "filter": "doggo = bernese"})).await;
+    snapshot!(response, @r###"
     {
-      "uid": 0,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "DOES_NOT_EXISTS",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -618,7 +573,7 @@ async fn delete_document_by_filter() {
         "originalFilter": "\"doggo = bernese\""
       },
       "error": {
-        "message": "Index `doggo` not found.",
+        "message": "Index `DOES_NOT_EXISTS` not found.",
         "code": "index_not_found",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -630,19 +585,14 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (response, code) = index.create(None).await;
-    snapshot!(code, @"202 Accepted");
-    server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-
     // no filterable are set
-    let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "doggo = bernese"})).await;
-    snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let index = shared_empty_index().await;
+    let (response, _code) =
+        index.delete_document_by_filter_fail(json!({ "filter": "doggo = bernese"})).await;
+    snapshot!(response, @r###"
     {
-      "uid": 2,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "EMPTY_INDEX",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -664,19 +614,16 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (response, code) = index.update_settings_filterable_attributes(json!(["doggo"])).await;
-    snapshot!(code, @"202 Accepted");
-    server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-
     // not filterable while there is a filterable attribute
+    let index = shared_index_with_documents().await;
     let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "catto = jorts"})).await;
+        index.delete_document_by_filter_fail(json!({ "filter": "catto = jorts"})).await;
     snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let response = server.wait_task(response.uid()).await;
+    snapshot!(response, @r###"
     {
-      "uid": 4,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "SHARED_DOCUMENTS",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -686,7 +633,7 @@ async fn delete_document_by_filter() {
         "originalFilter": "\"catto = jorts\""
       },
       "error": {
-        "message": "Attribute `catto` is not filterable. Available filterable attributes are: `doggo`.\n1:6 catto = jorts",
+        "message": "Attribute `catto` is not filterable. Available filterable attributes are: `id`, `title`.\n1:6 catto = jorts",
         "code": "invalid_document_filter",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_document_filter"
@@ -701,10 +648,10 @@ async fn delete_document_by_filter() {
 
 #[actix_rt::test]
 async fn fetch_document_by_filter() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     index.update_settings_filterable_attributes(json!(["color"])).await;
-    index
+    let (task, _code) = index
         .add_documents(
             json!([
                 { "id": 0, "color": "red" },
@@ -715,7 +662,7 @@ async fn fetch_document_by_filter() {
             Some("id"),
         )
         .await;
-    index.wait_task(1).await;
+    index.wait_task(task.uid()).await.succeeded();
 
     let (response, code) = index.get_document_by_filter(json!(null)).await;
     snapshot!(code, @"400 Bad Request");
@@ -798,8 +745,7 @@ async fn fetch_document_by_filter() {
 
 #[actix_rt::test]
 async fn retrieve_vectors() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let index = shared_empty_index().await;
 
     // GET ALL DOCUMENTS BY QUERY
     let (response, _code) = index.get_all_documents_raw("?retrieveVectors=tamo").await;

meilisearch/tests/dumps/mod.rs

@@ -2097,7 +2097,8 @@ async fn generate_and_import_dump_containing_vectors() {
           "source": "huggingFace",
           "model": "sentence-transformers/all-MiniLM-L6-v2",
           "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
-          "documentTemplate": "{{doc.doggo}}"
+          "documentTemplate": "{{doc.doggo}}",
+          "documentTemplateMaxBytes": 400
         }
       },
       "searchCutoffMs": null,

meilisearch/tests/index/errors.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?offset=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -21,7 +21,7 @@ async fn get_indexes_bad_offset() {
 
 #[actix_rt::test]
 async fn get_indexes_bad_limit() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?limit=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -37,7 +37,7 @@ async fn get_indexes_bad_limit() {
 
 #[actix_rt::test]
 async fn get_indexes_unknown_field() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?doggo=nolimit").await;
     snapshot!(code, @"400 Bad Request");
@@ -53,9 +53,9 @@ async fn get_indexes_unknown_field() {
 
 #[actix_rt::test]
 async fn create_index_missing_uid() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "primaryKey": "doggo" })).await;
+    let (response, code) = server.create_index_fail(json!({ "primaryKey": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -69,9 +69,9 @@ async fn create_index_missing_uid() {
 
 #[actix_rt::test]
 async fn create_index_bad_uid() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "uid": "the best doggo" })).await;
+    let (response, code) = server.create_index_fail(json!({ "uid": "the best doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -82,7 +82,7 @@ async fn create_index_bad_uid() {
     }
     "###);
 
-    let (response, code) = server.create_index(json!({ "uid": true })).await;
+    let (response, code) = server.create_index_fail(json!({ "uid": true })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -96,10 +96,10 @@ async fn create_index_bad_uid() {
 
 #[actix_rt::test]
 async fn create_index_bad_primary_key() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server
-        .create_index(json!({ "uid": "doggo", "primaryKey": ["the", "best", "doggo"] }))
+        .create_index_fail(json!({ "uid": "doggo", "primaryKey": ["the", "best", "doggo"] }))
         .await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -114,9 +114,10 @@ async fn create_index_bad_primary_key() {
 
 #[actix_rt::test]
 async fn create_index_unknown_field() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "uid": "doggo", "doggo": "bernese" })).await;
+    let (response, code) =
+        server.create_index_fail(json!({ "uid": "doggo", "doggo": "bernese" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -130,10 +131,8 @@ async fn create_index_unknown_field() {
 
 #[actix_rt::test]
 async fn get_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.get().await;
+    let server = Server::new_shared();
+    let (response, code) = server.get_index_fail("the good doggo").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -147,9 +146,8 @@ async fn get_index_bad_uid() {
 
 #[actix_rt::test]
 async fn update_index_bad_primary_key() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "primaryKey": ["doggo"] })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -164,9 +162,8 @@ async fn update_index_bad_primary_key() {
 
 #[actix_rt::test]
 async fn update_index_immutable_uid() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "uid": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -181,9 +178,8 @@ async fn update_index_immutable_uid() {
 
 #[actix_rt::test]
 async fn update_index_immutable_created_at() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "createdAt": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -198,9 +194,8 @@ async fn update_index_immutable_created_at() {
 
 #[actix_rt::test]
 async fn update_index_immutable_updated_at() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "updatedAt": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -215,9 +210,8 @@ async fn update_index_immutable_updated_at() {
 
 #[actix_rt::test]
 async fn update_index_unknown_field() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "doggo": "bork" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -232,10 +226,9 @@ async fn update_index_unknown_field() {
 
 #[actix_rt::test]
 async fn update_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.update_raw(json!({ "primaryKey": "doggo" })).await;
+    let server = Server::new_shared();
+    let (response, code) =
+        server.update_raw_index_fail("the good doggo", json!({ "primaryKey": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -249,10 +242,8 @@ async fn update_index_bad_uid() {
 
 #[actix_rt::test]
 async fn delete_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.delete().await;
+    let server = Server::new_shared();
+    let (response, code) = server.delete_index_fail("the good doggo").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {

meilisearch/tests/logs/error.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn logs_stream_bad_target() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) = server.service.post("/logs/stream", json!({ "target": true })).await;
@@ -58,7 +58,7 @@ async fn logs_stream_bad_target() {
 
 #[actix_rt::test]
 async fn logs_stream_bad_mode() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) = server.service.post("/logs/stream", json!({ "mode": true })).await;
@@ -99,7 +99,7 @@ async fn logs_stream_bad_mode() {
 
 #[actix_rt::test]
 async fn logs_stream_bad_profile_memory() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) =
@@ -156,7 +156,7 @@ async fn logs_stream_bad_profile_memory() {
 
 #[actix_rt::test]
 async fn logs_stream_without_enabling_the_route() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.service.post("/logs/stream", json!({})).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/logs/mod.rs

@@ -1,10 +1,13 @@
 mod error;
 
+use std::num::NonZeroUsize;
 use std::rc::Rc;
 use std::str::FromStr;
 
 use actix_web::http::header::ContentType;
+use actix_web::web::Data;
 use meili_snap::snapshot;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
 use tracing::level_filters::LevelFilter;
 use tracing_subscriber::layer::SubscriberExt;
@@ -40,10 +43,15 @@ async fn basic_test_log_stream_route() {
             .with_span_events(tracing_subscriber::fmt::format::FmtSpan::ACTIVE)
             .with_filter(tracing_subscriber::filter::LevelFilter::from_str("OFF").unwrap()),
     );
+    let search_queue = SearchQueue::new(
+        server.service.options.experimental_search_queue_size,
+        NonZeroUsize::new(1).unwrap(),
+    );
 
     let app = actix_web::test::init_service(create_app(
         server.service.index_scheduler.clone().into(),
         server.service.auth.clone().into(),
+        Data::new(search_queue),
         server.service.options.clone(),
         (route_layer_handle, stderr_layer_handle),
         analytics::MockAnalytics::new(&server.service.options),

meilisearch/tests/search/errors.rs

@@ -1,16 +1,13 @@
 use meili_snap::*;
 
-use super::DOCUMENTS;
-use crate::common::Server;
+use crate::common::{shared_does_not_exists_index, Server};
 use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let index = shared_does_not_exists_index().await;
     let expected_response = json!({
-        "message": "Index `test` not found.",
+        "message": "Index `DOES_NOT_EXISTS` not found.",
         "code": "index_not_found",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -26,8 +23,8 @@ async fn search_unexisting_index() {
 
 #[actix_rt::test]
 async fn search_unexisting_parameter() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     index
         .search(json!({"marin": "hello"}), |response, code| {
@@ -39,8 +36,8 @@ async fn search_unexisting_parameter() {
 
 #[actix_rt::test]
 async fn search_bad_q() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"q": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -57,8 +54,8 @@ async fn search_bad_q() {
 
 #[actix_rt::test]
 async fn search_bad_offset() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"offset": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -85,8 +82,8 @@ async fn search_bad_offset() {
 
 #[actix_rt::test]
 async fn search_bad_limit() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"limit": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -113,8 +110,8 @@ async fn search_bad_limit() {
 
 #[actix_rt::test]
 async fn search_bad_page() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"page": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -141,8 +138,8 @@ async fn search_bad_page() {
 
 #[actix_rt::test]
 async fn search_bad_hits_per_page() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"hitsPerPage": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -169,8 +166,8 @@ async fn search_bad_hits_per_page() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_retrieve() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToRetrieve": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -187,8 +184,8 @@ async fn search_bad_attributes_to_retrieve() {
 
 #[actix_rt::test]
 async fn search_bad_retrieve_vectors() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"retrieveVectors": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -237,8 +234,8 @@ async fn search_bad_retrieve_vectors() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_crop() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToCrop": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -255,8 +252,8 @@ async fn search_bad_attributes_to_crop() {
 
 #[actix_rt::test]
 async fn search_bad_crop_length() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"cropLength": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -283,8 +280,8 @@ async fn search_bad_crop_length() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_highlight() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToHighlight": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -303,14 +300,12 @@ async fn search_bad_attributes_to_highlight() {
 async fn search_bad_filter() {
     // Since a filter is deserialized as a json Value it will never fail to deserialize.
     // Thus the error message is not generated by deserr but written by us.
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     // Also, to trigger the error message we need to effectively create the index or else it'll throw an
     // index does not exists error.
-    let (_, code) = index.create(None).await;
-    server.wait_task(0).await;
-
-    snapshot!(code, @"202 Accepted");
+    let (response, _code) = index.create(None).await;
+    server.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({ "filter": true })).await;
     snapshot!(code, @"400 Bad Request");
@@ -327,8 +322,8 @@ async fn search_bad_filter() {
 
 #[actix_rt::test]
 async fn search_bad_sort() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"sort": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -345,8 +340,8 @@ async fn search_bad_sort() {
 
 #[actix_rt::test]
 async fn search_bad_show_matches_position() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"showMatchesPosition": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -373,8 +368,8 @@ async fn search_bad_show_matches_position() {
 
 #[actix_rt::test]
 async fn search_bad_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"facets": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -391,8 +386,8 @@ async fn search_bad_facets() {
 
 #[actix_rt::test]
 async fn search_bad_threshold() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"rankingScoreThreshold": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -408,8 +403,8 @@ async fn search_bad_threshold() {
 
 #[actix_rt::test]
 async fn search_invalid_threshold() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"rankingScoreThreshold": 42})).await;
     snapshot!(code, @"400 Bad Request");
@@ -425,11 +420,11 @@ async fn search_invalid_threshold() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     // Wait for the settings update to complete
-    index.wait_task(0).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -456,10 +451,11 @@ async fn search_non_filterable_facets() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_multiple_filterable() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -486,10 +482,10 @@ async fn search_non_filterable_facets_multiple_filterable() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_no_filterable() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": []})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"filterableAttributes": []})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -516,10 +512,11 @@ async fn search_non_filterable_facets_no_filterable() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_multiple_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _uid) =
+        index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo", "neko"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -546,8 +543,8 @@ async fn search_non_filterable_facets_multiple_facets() {
 
 #[actix_rt::test]
 async fn search_bad_highlight_pre_tag() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"highlightPreTag": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -564,8 +561,8 @@ async fn search_bad_highlight_pre_tag() {
 
 #[actix_rt::test]
 async fn search_bad_highlight_post_tag() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"highlightPostTag": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -582,8 +579,8 @@ async fn search_bad_highlight_post_tag() {
 
 #[actix_rt::test]
 async fn search_bad_crop_marker() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"cropMarker": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -600,8 +597,8 @@ async fn search_bad_crop_marker() {
 
 #[actix_rt::test]
 async fn search_bad_matching_strategy() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"matchingStrategy": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -639,13 +636,10 @@ async fn search_bad_matching_strategy() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_object() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    let (task, _code) = index.add_documents(documents, None).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     index.wait_task(task.uid()).await;
 
     index
@@ -665,13 +659,10 @@ async fn filter_invalid_syntax_object() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    let (task, _code) = index.add_documents(documents, None).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     index.wait_task(task.uid()).await;
 
     index
@@ -691,14 +682,11 @@ async fn filter_invalid_syntax_array() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Found unexpected characters at the end of the filter: `XOR title = Glass`. You probably forgot an `OR` or an `AND` rule.\n15:32 title = Glass XOR title = Glass",
@@ -716,14 +704,11 @@ async fn filter_invalid_syntax_string() {
 
 #[actix_rt::test]
 async fn filter_invalid_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.\n1:5 many = Glass",
@@ -741,14 +726,11 @@ async fn filter_invalid_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_invalid_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.\n1:5 many = Glass",
@@ -766,14 +748,11 @@ async fn filter_invalid_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:13 _geo = Glass",
@@ -791,14 +770,11 @@ async fn filter_reserved_geo_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:13 _geo = Glass",
@@ -816,14 +792,11 @@ async fn filter_reserved_geo_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:21 _geoDistance = Glass",
@@ -841,14 +814,11 @@ async fn filter_reserved_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
        "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:21 _geoDistance = Glass",
@@ -866,14 +836,11 @@ async fn filter_reserved_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_point_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:18 _geoPoint = Glass",
@@ -891,14 +858,11 @@ async fn filter_reserved_geo_point_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_point_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
        "message": "`_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:18 _geoPoint = Glass",
@@ -916,14 +880,11 @@ async fn filter_reserved_geo_point_string() {
 
 #[actix_rt::test]
 async fn sort_geo_reserved_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a sort expression. Use the _geoPoint(latitude, longitude) built-in rule to sort on _geo field coordinates.",
@@ -946,14 +907,11 @@ async fn sort_geo_reserved_attribute() {
 
 #[actix_rt::test]
 async fn sort_reserved_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoDistance` is a reserved keyword and thus can't be used as a sort expression.",
@@ -976,14 +934,10 @@ async fn sort_reserved_attribute() {
 
 #[actix_rt::test]
 async fn sort_unsortable_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "Attribute `title` is not sortable. Available sortable attributes are: `id`.",
@@ -1006,14 +960,11 @@ async fn sort_unsortable_attribute() {
 
 #[actix_rt::test]
 async fn sort_invalid_syntax() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (response, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "Invalid syntax for the sort parameter: expected expression ending by `:asc` or `:desc`, found `title`.",
@@ -1036,18 +987,15 @@ async fn sort_invalid_syntax() {
 
 #[actix_rt::test]
 async fn sort_unset_ranking_rule() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index
+    let (response, _code) = index
         .update_settings(
             json!({"sortableAttributes": ["title"], "rankingRules": ["proximity", "exactness"]}),
         )
         .await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "You must specify where `sort` is listed in the rankingRules setting to use the sort parameter at search time.",
@@ -1070,14 +1018,11 @@ async fn sort_unset_ranking_rule() {
 
 #[actix_rt::test]
 async fn search_on_unknown_field() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings_searchable_attributes(json!(["id", "title"])).await;
-    index.wait_task(0).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings_searchable_attributes(json!(["id", "title"])).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     index
         .search(
@@ -1099,14 +1044,11 @@ async fn search_on_unknown_field() {
 
 #[actix_rt::test]
 async fn search_on_unknown_field_plus_joker() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings_searchable_attributes(json!(["id", "title"])).await;
-    index.wait_task(0).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings_searchable_attributes(json!(["id", "title"])).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     index
         .search(
@@ -1145,11 +1087,10 @@ async fn search_on_unknown_field_plus_joker() {
 
 #[actix_rt::test]
 async fn distinct_at_search_time() {
-    let server = Server::new().await;
-    let index = server.index("tamo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (task, _) = index.create(None).await;
-    let task = index.wait_task(task.uid()).await;
-    snapshot!(task, name: "task-succeed");
+    index.wait_task(task.uid()).await.succeeded();
 
     let (response, code) =
         index.search_post(json!({"page": 0, "hitsPerPage": 2, "distinct": "doggo.truc"})).await;
@@ -1210,8 +1151,8 @@ async fn distinct_at_search_time() {
 async fn search_with_contains_without_enabling_the_feature() {
     // Since a filter is deserialized as a json Value it will never fail to deserialize.
     // Thus the error message is not generated by deserr but written by us.
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     // Also, to trigger the error message we need to effectively create the index or else it'll throw an
     // index does not exists error.
     let (task, _code) = index.create(None).await;

meilisearch/tests/search/mod.rs

@@ -16,208 +16,17 @@ mod restrict_searchable;
 mod search_queue;
 
 use meilisearch::Opt;
-use once_cell::sync::Lazy;
 use tempfile::TempDir;
 
-use crate::common::{default_settings, Server, Value};
+use crate::common::{
+    default_settings, shared_index_with_documents, shared_index_with_nested_documents, Server,
+    DOCUMENTS, FRUITS_DOCUMENTS, NESTED_DOCUMENTS, SCORE_DOCUMENTS, VECTOR_DOCUMENTS,
+};
 use crate::json;
 
-static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Shazam!",
-            "id": "287947",
-            "_vectors": { "manual": [1, 2, 3]},
-        },
-        {
-            "title": "Captain Marvel",
-            "id": "299537",
-            "_vectors": { "manual": [1, 2, 54] },
-        },
-        {
-            "title": "Escape Room",
-            "id": "522681",
-            "_vectors": { "manual": [10, -23, 32] },
-        },
-        {
-            "title": "How to Train Your Dragon: The Hidden World",
-            "id": "166428",
-            "_vectors": { "manual": [-100, 231, 32] },
-        },
-        {
-            "title": "Gläss",
-            "id": "450465",
-            "_vectors": { "manual": [-100, 340, 90] },
-        }
-    ])
-});
-
-static SCORE_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Batman the dark knight returns: Part 1",
-            "id": "A",
-        },
-        {
-            "title": "Batman the dark knight returns: Part 2",
-            "id": "B",
-        },
-        {
-            "title": "Batman Returns",
-            "id": "C",
-        },
-        {
-            "title": "Batman",
-            "id": "D",
-        },
-        {
-            "title": "Badman",
-            "id": "E",
-        }
-    ])
-});
-
-static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "id": 852,
-            "father": "jean",
-            "mother": "michelle",
-            "doggos": [
-                {
-                    "name": "bobby",
-                    "age": 2,
-                },
-                {
-                    "name": "buddy",
-                    "age": 4,
-                },
-            ],
-            "cattos": "pésti",
-            "_vectors": { "manual": [1, 2, 3]},
-        },
-        {
-            "id": 654,
-            "father": "pierre",
-            "mother": "sabine",
-            "doggos": [
-                {
-                    "name": "gros bill",
-                    "age": 8,
-                },
-            ],
-            "cattos": ["simba", "pestiféré"],
-            "_vectors": { "manual": [1, 2, 54] },
-        },
-        {
-            "id": 750,
-            "father": "romain",
-            "mother": "michelle",
-            "cattos": ["enigma"],
-            "_vectors": { "manual": [10, 23, 32] },
-        },
-        {
-            "id": 951,
-            "father": "jean-baptiste",
-            "mother": "sophie",
-            "doggos": [
-                {
-                    "name": "turbo",
-                    "age": 5,
-                },
-                {
-                    "name": "fast",
-                    "age": 6,
-                },
-            ],
-            "cattos": ["moumoute", "gomez"],
-            "_vectors": { "manual": [10, 23, 32] },
-        },
-    ])
-});
-
-static FRUITS_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "name": "Exclusive sale: green apple",
-            "id": "green-apple-boosted",
-            "BOOST": true
-        },
-        {
-            "name": "Pear",
-            "id": "pear",
-        },
-        {
-            "name": "Red apple gala",
-            "id": "red-apple-gala",
-        },
-        {
-            "name": "Exclusive sale: Red Tomato",
-            "id": "red-tomatoes-boosted",
-            "BOOST": true
-        },
-        {
-            "name": "Exclusive sale: Red delicious apple",
-            "id": "red-delicious-boosted",
-            "BOOST": true,
-        }
-    ])
-});
-
-static VECTOR_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-      {
-        "id": "A",
-        "description": "the dog barks at the cat",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.9, 0.8, 0.05],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-0.1, 0.55]
-        }
-      },
-      {
-        "id": "B",
-        "description": "the kitten scratched the beagle",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.8, 0.9, 0.5],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-0.2, 0.65]
-        }
-      },
-      {
-        "id": "C",
-        "description": "the dog had to stay alone today",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.85, 0.02, 0.1],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-1.0, 0.1]
-        }
-      },
-      {
-        "id": "D",
-        "description": "the little boy pets the puppy",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.8, 0.09, 0.8],
-          // dimensions [negative/positive, energy]
-          "sentiment": [0.8, 0.3]
-        }
-      },
-    ])
-});
-
 #[actix_rt::test]
 async fn simple_placeholder_search() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -225,11 +34,7 @@ async fn simple_placeholder_search() {
         })
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -240,13 +45,7 @@ async fn simple_placeholder_search() {
 
 #[actix_rt::test]
 async fn simple_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "glass"}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -254,11 +53,7 @@ async fn simple_search() {
         })
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(json!({"q": "pésti"}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -290,13 +85,7 @@ async fn phrase_search_with_stop_word() {
 
 #[actix_rt::test]
 async fn negative_phrase_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "-\"train your dragon\"" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -312,13 +101,7 @@ async fn negative_phrase_search() {
 
 #[actix_rt::test]
 async fn negative_word_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "-escape" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -343,13 +126,7 @@ async fn negative_word_search() {
 
 #[actix_rt::test]
 async fn non_negative_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "- escape" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -442,13 +219,7 @@ async fn test_thai_language() {
 
 #[actix_rt::test]
 async fn search_multiple_params() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -465,11 +236,7 @@ async fn search_multiple_params() {
         )
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -555,15 +322,7 @@ async fn search_with_filter_string_notation() {
 
 #[actix_rt::test]
 async fn search_with_filter_array_notation() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     let (response, code) = index
         .search_post(json!({
             "filter": ["title = Gläss"]
@@ -609,15 +368,7 @@ async fn search_with_contains_filter() {
 
 #[actix_rt::test]
 async fn search_with_sort_on_numbers() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -630,14 +381,7 @@ async fn search_with_sort_on_numbers() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"sortableAttributes": ["doggos.age"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -653,15 +397,7 @@ async fn search_with_sort_on_numbers() {
 
 #[actix_rt::test]
 async fn search_with_sort_on_strings() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -674,14 +410,7 @@ async fn search_with_sort_on_strings() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"sortableAttributes": ["doggos.name"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -697,15 +426,7 @@ async fn search_with_sort_on_strings() {
 
 #[actix_rt::test]
 async fn search_with_multiple_sort() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id", "title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     let (response, code) = index
         .search_post(json!({
             "sort": ["id:asc", "title:desc"]
@@ -717,15 +438,7 @@ async fn search_with_multiple_sort() {
 
 #[actix_rt::test]
 async fn search_facet_distribution() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -740,13 +453,7 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"filterableAttributes": ["father", "doggos.name"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
+    let index = shared_index_with_nested_documents().await;
 
     // TODO: TAMO: fix the test
     index
@@ -773,9 +480,6 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    index.update_settings(json!({"filterableAttributes": ["doggos"]})).await;
-    index.wait_task(4).await;
-
     index
         .search(
             json!({
@@ -811,9 +515,6 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    index.update_settings(json!({"filterableAttributes": ["doggos.name"]})).await;
-    index.wait_task(5).await;
-
     index
         .search(
             json!({
@@ -1043,6 +744,10 @@ async fn test_score_details() {
                   {
                     "title": "How to Train Your Dragon: The Hidden World",
                     "id": "166428",
+                    "color": [
+                      "green",
+                      "red"
+                    ],
                     "_vectors": {
                       "manual": [
                         -100.0,
@@ -1469,6 +1174,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Shazam!",
         "id": "287947",
+        "color": [
+          "green",
+          "blue"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1486,6 +1195,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Captain Marvel",
         "id": "299537",
+        "color": [
+          "yellow",
+          "blue"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1503,6 +1216,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Gläss",
         "id": "450465",
+        "color": [
+          "blue",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1520,6 +1237,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "How to Train Your Dragon: The Hidden World",
         "id": "166428",
+        "color": [
+          "green",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1537,6 +1258,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Escape Room",
         "id": "522681",
+        "color": [
+          "yellow",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1803,6 +1528,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,
@@ -1824,6 +1553,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,
@@ -1845,6 +1578,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,

meilisearch/tests/search/multi.rs

@@ -94,7 +94,7 @@ async fn simple_search_single_index() {
 
     let (response, code) = server
         .multi_search(json!({"queries": [
-        {"indexUid" : "test", "q": "glass"},
+        {"indexUid": "test", "q": "glass"},
         {"indexUid": "test", "q": "captain"},
         ]}))
         .await;
@@ -107,6 +107,10 @@ async fn simple_search_single_index() {
           {
             "title": "Gläss",
             "id": "450465",
+            "color": [
+              "blue",
+              "red"
+            ],
             "_vectors": {
               "manual": [
                 -100.0,
@@ -128,6 +132,10 @@ async fn simple_search_single_index() {
           {
             "title": "Captain Marvel",
             "id": "299537",
+            "color": [
+              "yellow",
+              "blue"
+            ],
             "_vectors": {
               "manual": [
                 1.0,
@@ -168,6 +176,10 @@ async fn federation_single_search_single_index() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -287,6 +299,10 @@ async fn federation_two_search_single_index() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -303,6 +319,10 @@ async fn federation_two_search_single_index() {
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -454,6 +474,10 @@ async fn simple_search_two_indexes() {
           {
             "title": "Gläss",
             "id": "450465",
+            "color": [
+              "blue",
+              "red"
+            ],
             "_vectors": {
               "manual": [
                 -100.0,
@@ -555,6 +579,10 @@ async fn federation_two_search_two_indexes() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -673,6 +701,10 @@ async fn federation_multiple_search_multiple_indexes() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -735,6 +767,10 @@ async fn federation_multiple_search_multiple_indexes() {
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -751,6 +787,10 @@ async fn federation_multiple_search_multiple_indexes() {
         {
           "title": "Escape Room",
           "id": "522681",
+          "color": [
+            "yellow",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               10.0,
@@ -852,6 +892,10 @@ async fn federation_multiple_search_multiple_indexes() {
         {
           "title": "How to Train Your Dragon: The Hidden World",
           "id": "166428",
+          "color": [
+            "green",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2053,6 +2097,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2070,6 +2118,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "Escape Room",
           "id": "522681",
+          "color": [
+            "yellow",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               10.0,
@@ -2087,6 +2139,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2104,6 +2160,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "How to Train Your Dragon: The Hidden World",
           "id": "166428",
+          "color": [
+            "green",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2121,6 +2181,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "Shazam!",
           "id": "287947",
+          "color": [
+            "green",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2158,6 +2222,10 @@ async fn federation_sort_different_indexes_same_criterion_same_direction() {
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2341,6 +2409,10 @@ async fn federation_sort_different_ranking_rules() {
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2358,6 +2430,10 @@ async fn federation_sort_different_ranking_rules() {
         {
           "title": "Escape Room",
           "id": "522681",
+          "color": [
+            "yellow",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               10.0,
@@ -2375,6 +2451,10 @@ async fn federation_sort_different_ranking_rules() {
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2392,6 +2472,10 @@ async fn federation_sort_different_ranking_rules() {
         {
           "title": "How to Train Your Dragon: The Hidden World",
           "id": "166428",
+          "color": [
+            "green",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2409,6 +2493,10 @@ async fn federation_sort_different_ranking_rules() {
         {
           "title": "Shazam!",
           "id": "287947",
+          "color": [
+            "green",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2622,6 +2710,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2659,6 +2751,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "Escape Room",
           "id": "522681",
+          "color": [
+            "yellow",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               10.0,
@@ -2676,6 +2772,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "Gläss",
           "id": "450465",
+          "color": [
+            "blue",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2693,6 +2793,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "How to Train Your Dragon: The Hidden World",
           "id": "166428",
+          "color": [
+            "green",
+            "red"
+          ],
           "_vectors": {
             "manual": [
               -100.0,
@@ -2710,6 +2814,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "Shazam!",
           "id": "287947",
+          "color": [
+            "green",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,
@@ -2767,6 +2875,10 @@ async fn federation_sort_different_indexes_different_criterion_same_direction()
         {
           "title": "Captain Marvel",
           "id": "299537",
+          "color": [
+            "yellow",
+            "blue"
+          ],
           "_vectors": {
             "manual": [
               1.0,

meilisearch/tests/search/pagination.rs

@@ -1,16 +1,10 @@
+use super::shared_index_with_documents;
 use crate::common::Server;
 use crate::json;
-use crate::search::DOCUMENTS;
 
 #[actix_rt::test]
 async fn default_search_should_return_estimated_total_hit() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -28,13 +22,7 @@ async fn default_search_should_return_estimated_total_hit() {
 
 #[actix_rt::test]
 async fn simple_search() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"page": 1}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -53,13 +41,7 @@ async fn simple_search() {
 
 #[actix_rt::test]
 async fn page_zero_should_not_return_any_result() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"page": 0}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -73,13 +55,7 @@ async fn page_zero_should_not_return_any_result() {
 
 #[actix_rt::test]
 async fn hits_per_page_1() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"hitsPerPage": 1}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -93,13 +69,7 @@ async fn hits_per_page_1() {
 
 #[actix_rt::test]
 async fn hits_per_page_0_should_not_return_any_result() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"hitsPerPage": 0}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -113,8 +83,8 @@ async fn hits_per_page_0_should_not_return_any_result() {
 
 #[actix_rt::test]
 async fn ensure_placeholder_search_hit_count_valid() {
-    let server = Server::new().await;
-    let index = server.index("basic");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let documents = json!([
         {
@@ -143,15 +113,15 @@ async fn ensure_placeholder_search_hit_count_valid() {
             "distinct": 3,
         }
     ]);
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.add_documents(documents, None).await;
+    index.wait_task(task.uid()).await.succeeded();
 
-    let (_response, _code) = index
+    let (response, _code) = index
         .update_settings(
             json!({ "rankingRules": ["distinct:asc"], "distinctAttribute": "distinct"}),
         )
         .await;
-    index.wait_task(1).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     for page in 0..=4 {
         index

meilisearch/tests/search/restrict_searchable.rs

@@ -8,8 +8,8 @@ use crate::json;
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");
 
-    index.add_documents(documents.clone(), None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.add_documents(documents.clone(), None).await;
+    index.wait_task(task.uid()).await.succeeded();
     index
 }
 

meilisearch/tests/search/search_queue.rs

@@ -37,6 +37,43 @@ async fn search_queue_register() {
         .unwrap();
 }
 
+#[actix_rt::test]
+async fn search_queue_register_with_explicit_drop() {
+    let queue = SearchQueue::new(4, NonZeroUsize::new(2).unwrap());
+
+    // First, use all the cores
+    let permit1 = queue.try_get_search_permit().await.unwrap();
+    let _permit2 = queue.try_get_search_permit().await.unwrap();
+
+    // If we free one spot we should be able to register one new search
+    permit1.drop().await;
+
+    let permit3 = queue.try_get_search_permit().await.unwrap();
+
+    // And again
+    permit3.drop().await;
+
+    let _permit4 = queue.try_get_search_permit().await.unwrap();
+}
+
+#[actix_rt::test]
+async fn search_queue_register_with_time_to_abort() {
+    let queue = Arc::new(
+        SearchQueue::new(1, NonZeroUsize::new(1).unwrap())
+            .with_time_to_abort(Duration::from_secs(1)),
+    );
+
+    // First, use all the cores
+    let permit1 = queue.try_get_search_permit().await.unwrap();
+    let q = queue.clone();
+    let permit2 = tokio::task::spawn(async move { q.try_get_search_permit().await });
+    tokio::time::sleep(Duration::from_secs(1)).await;
+    permit1.drop().await;
+    let ret = permit2.await.unwrap();
+
+    snapshot!(ret.unwrap_err(), @"Too many search requests running at the same time: 1. Retry after 10s.");
+}
+
 #[actix_rt::test]
 async fn wait_till_cores_are_available() {
     let queue = Arc::new(SearchQueue::new(4, NonZeroUsize::new(1).unwrap()));

meilisearch/tests/settings/errors.rs

@@ -5,8 +5,8 @@ use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "displayedAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -33,8 +33,8 @@ async fn settings_bad_displayed_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_searchable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "searchableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -61,8 +61,8 @@ async fn settings_bad_searchable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_filterable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "filterableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -89,8 +89,8 @@ async fn settings_bad_filterable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_sortable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "sortableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -117,8 +117,8 @@ async fn settings_bad_sortable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_ranking_rules() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "rankingRules": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -145,8 +145,8 @@ async fn settings_bad_ranking_rules() {
 
 #[actix_rt::test]
 async fn settings_bad_stop_words() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "stopWords": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -173,8 +173,8 @@ async fn settings_bad_stop_words() {
 
 #[actix_rt::test]
 async fn settings_bad_synonyms() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "synonyms": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -201,8 +201,8 @@ async fn settings_bad_synonyms() {
 
 #[actix_rt::test]
 async fn settings_bad_distinct_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "distinctAttribute": ["doggo"] })).await;
     snapshot!(code, @"400 Bad Request");
@@ -229,8 +229,8 @@ async fn settings_bad_distinct_attribute() {
 
 #[actix_rt::test]
 async fn settings_bad_typo_tolerance() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "typoTolerance": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -284,8 +284,8 @@ async fn settings_bad_typo_tolerance() {
 
 #[actix_rt::test]
 async fn settings_bad_faceting() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "faceting": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -312,8 +312,8 @@ async fn settings_bad_faceting() {
 
 #[actix_rt::test]
 async fn settings_bad_pagination() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "pagination": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -340,8 +340,8 @@ async fn settings_bad_pagination() {
 
 #[actix_rt::test]
 async fn settings_bad_search_cutoff_ms() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "searchCutoffMs": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/settings/get_settings.rs

@@ -190,7 +190,8 @@ async fn secrets_are_hidden_in_settings() {
           "source": "rest",
           "apiKey": "My suXXXXXX...",
           "dimensions": 4,
-          "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+          "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+          "documentTemplateMaxBytes": 400,
           "url": "https://localhost:7777",
           "request": "{{text}}",
           "response": "{{embedding}}",

meilisearch/tests/swap_indexes/errors.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.index_swap(json!("doggo")).await;
     snapshot!(code, @"400 Bad Request");
@@ -32,7 +32,7 @@ async fn swap_indexes_bad_format() {
 
 #[actix_rt::test]
 async fn swap_indexes_bad_indexes() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.index_swap(json!([{ "indexes": "doggo"}])).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/tasks/errors.rs

@@ -4,7 +4,7 @@ use crate::common::Server;
 
 #[actix_rt::test]
 async fn task_bad_uids() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("uids=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -53,7 +53,7 @@ async fn task_bad_uids() {
 
 #[actix_rt::test]
 async fn task_bad_canceled_by() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("canceledBy=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -91,7 +91,7 @@ async fn task_bad_canceled_by() {
 
 #[actix_rt::test]
 async fn task_bad_types() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("types=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -129,7 +129,7 @@ async fn task_bad_types() {
 
 #[actix_rt::test]
 async fn task_bad_statuses() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("statuses=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -167,7 +167,7 @@ async fn task_bad_statuses() {
 
 #[actix_rt::test]
 async fn task_bad_index_uids() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("indexUids=the%20good%20doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -205,7 +205,7 @@ async fn task_bad_index_uids() {
 
 #[actix_rt::test]
 async fn task_bad_limit() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("limit=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -243,7 +243,7 @@ async fn task_bad_limit() {
 
 #[actix_rt::test]
 async fn task_bad_from() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("from=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -281,7 +281,7 @@ async fn task_bad_from() {
 
 #[actix_rt::test]
 async fn task_bad_after_enqueued_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterEnqueuedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -319,7 +319,7 @@ async fn task_bad_after_enqueued_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_enqueued_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeEnqueuedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -357,7 +357,7 @@ async fn task_bad_before_enqueued_at() {
 
 #[actix_rt::test]
 async fn task_bad_after_started_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterStartedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -395,7 +395,7 @@ async fn task_bad_after_started_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_started_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeStartedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -433,7 +433,7 @@ async fn task_bad_before_started_at() {
 
 #[actix_rt::test]
 async fn task_bad_after_finished_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterFinishedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -471,7 +471,7 @@ async fn task_bad_after_finished_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_finished_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeFinishedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/tasks/mod.rs

@@ -66,8 +66,8 @@ async fn list_tasks() {
 async fn list_tasks_with_star_filters() {
     let server = Server::new().await;
     let index = server.index("test");
-    index.create(None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.create(None).await;
+    index.wait_task(task.uid()).await;
     index
         .add_documents(serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), None)
         .await;

meilisearch/tests/vector/mod.rs

@@ -2,10 +2,13 @@ mod openai;
 mod rest;
 mod settings;
 
+use std::str::FromStr;
+
 use meili_snap::{json_string, snapshot};
+use meilisearch::option::MaxThreads;
 
 use crate::common::index::Index;
-use crate::common::{GetAllDocumentsOptions, Server};
+use crate::common::{default_settings, GetAllDocumentsOptions, Server};
 use crate::json;
 
 async fn get_server_vector() -> Server {
@@ -503,7 +506,11 @@ async fn user_provided_embeddings_error() {
 
 #[actix_rt::test]
 async fn user_provided_vectors_error() {
-    let server = Server::new().await;
+    let temp = tempfile::tempdir().unwrap();
+    let mut options = default_settings(temp.path());
+    // If we have more than one indexing thread the error messages below may become inconsistent
+    options.indexer_options.max_indexing_threads = MaxThreads::from_str("1").unwrap();
+    let server = Server::new_with_options(options).await.unwrap();
 
     let index = generate_default_user_provided_documents(&server).await;
 

meilisearch/tests/vector/openai.rs

@@ -302,7 +302,8 @@ async fn create_mock_with_template(
         "source": "openAi",
         "url": url,
         "apiKey": API_KEY,
-        "documentTemplate": document_template
+        "documentTemplate": document_template,
+        "documentTemplateMaxBytes": 8000000,
     });
 
     model_dimensions.add_to_settings(&mut embedder_settings);
@@ -693,6 +694,7 @@ async fn bad_api_key() {
             "model": "text-embedding-3-large",
             "apiKey": "XXX...",
             "documentTemplate": "{%- if doc.gender == \"F\" -%}Une chienne nommée {{doc.name}}, née en {{doc.birthyear}}\n        {%- else -%}\n        Un chien nommé {{doc.name}}, né en {{doc.birthyear}}\n        {%- endif %}, de race {{doc.breed}}.",
+            "documentTemplateMaxBytes": 8000000,
             "url": "[url]"
           }
         }
@@ -735,6 +737,7 @@ async fn bad_api_key() {
             "source": "openAi",
             "model": "text-embedding-3-large",
             "documentTemplate": "{%- if doc.gender == \"F\" -%}Une chienne nommée {{doc.name}}, née en {{doc.birthyear}}\n        {%- else -%}\n        Un chien nommé {{doc.name}}, né en {{doc.birthyear}}\n        {%- endif %}, de race {{doc.breed}}.",
+            "documentTemplateMaxBytes": 8000000,
             "url": "[url]"
           }
         }

meilisearch/tests/vector/rest.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use meili_snap::{json_string, snapshot};
@@ -37,6 +38,46 @@ async fn create_mock() -> (MockServer, Value) {
     (mock_server, embedder_settings)
 }
 
+async fn create_mock_map() -> (MockServer, Value) {
+    let mock_server = MockServer::start().await;
+
+    let text_to_embedding: BTreeMap<_, _> = vec![
+        // text -> embedding
+        ("name: kefir\n", [0.0, 0.1, 0.2]),
+    ]
+    // turn into btree
+    .into_iter()
+    .collect();
+
+    Mock::given(method("POST"))
+        .and(path("/"))
+        .respond_with(move |req: &Request| {
+            let text: String = req.body_json().unwrap();
+            match text_to_embedding.get(text.as_str()) {
+                Some(embedding) => {
+                    ResponseTemplate::new(200).set_body_json(json!({ "data": embedding }))
+                }
+                None => ResponseTemplate::new(404)
+                    .set_body_json(json!({"error": "text not found", "text": text})),
+            }
+        })
+        .mount(&mock_server)
+        .await;
+    let url = mock_server.uri();
+
+    let embedder_settings = json!({
+        "source": "rest",
+        "url": url,
+        "dimensions": 3,
+        "request": "{{text}}",
+        "response": {
+          "data": "{{embedding}}"
+        }
+    });
+
+    (mock_server, embedder_settings)
+}
+
 #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
 struct MultipleRequest {
     input: Vec<String>,
@@ -1100,6 +1141,7 @@ async fn server_returns_bad_request() {
 
     let (response, code) = index
         .update_settings(json!({
+          "searchableAttributes": ["name", "missing_field"],
           "embedders": {
               "rest": json!({ "source": "rest", "url": mock.uri(), "request": "{{text}}", "response": "{{embedding}}", "dimensions": 3 }),
           },
@@ -1115,6 +1157,10 @@ async fn server_returns_bad_request() {
       "type": "settingsUpdate",
       "canceledBy": null,
       "details": {
+        "searchableAttributes": [
+          "name",
+          "missing_field"
+        ],
         "embedders": {
           "rest": {
             "source": "rest",
@@ -1148,7 +1194,7 @@ async fn server_returns_bad_request() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "While embedding documents for embedder `rest`: user error: sent a bad request to embedding server\n  - Hint: check that the `request` in the embedder configuration matches the remote server's API\n  - server replied with `{\"error\":\"Invalid request: invalid type: string \\\" id: 1\\\\n name: kefir\\\\n\\\", expected struct MultipleRequest at line 1 column 24\"}`",
+        "message": "While embedding documents for embedder `rest`: user error: sent a bad request to embedding server\n  - Hint: check that the `request` in the embedder configuration matches the remote server's API\n  - server replied with `{\"error\":\"Invalid request: invalid type: string \\\"name: kefir\\\\n\\\", expected struct MultipleRequest at line 1 column 15\"}`",
         "code": "vector_embedding_error",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
@@ -1891,3 +1937,109 @@ async fn server_custom_header() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn searchable_reindex() {
+    let (_mock, setting) = create_mock_map().await;
+    let server = get_server_vector().await;
+    let index = server.index("doggo");
+
+    let (response, code) = index
+        .update_settings(json!({
+          "searchableAttributes": ["name", "missing_field"],
+          "embedders": {
+              "rest": setting,
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "succeeded",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "searchableAttributes": [
+          "name",
+          "missing_field"
+        ],
+        "embedders": {
+          "rest": {
+            "source": "rest",
+            "dimensions": 3,
+            "url": "[url]",
+            "request": "{{text}}",
+            "response": {
+              "data": "{{embedding}}"
+            }
+          }
+        }
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    let (response, code) =
+        index.add_documents(json!( { "id": 1, "name": "kefir", "breed": "patou" }), None).await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "succeeded",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 1
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    // triggers reindexing with the new searchable attribute.
+    // as the mock intentionally doesn't know of this text, the task will fail, outputting the putative rendered text.
+    let (response, code) = index
+        .update_settings(json!({
+          "searchableAttributes": ["breed"],
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "failed",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "searchableAttributes": [
+          "breed"
+        ]
+      },
+      "error": {
+        "message": "While embedding documents for embedder `rest`: error: received unexpected HTTP 404 from embedding server\n  - server replied with `{\"error\":\"text not found\",\"text\":\"breed: patou\\n\"}`",
+        "code": "vector_embedding_error",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+}

meilitool/Cargo.toml

@@ -15,5 +15,6 @@ dump = { path = "../dump" }
 file-store = { path = "../file-store" }
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
+serde = { version = "1.0.209", features = ["derive"] }
 time = { version = "0.3.36", features = ["formatting"] }
 uuid = { version = "1.10.0", features = ["v4"], default-features = false }

meilitool/src/main.rs

@@ -2,7 +2,7 @@ use std::fs::{read_dir, read_to_string, remove_file, File};
 use std::io::BufWriter;
 use std::path::PathBuf;
 
-use anyhow::Context;
+use anyhow::{bail, Context};
 use clap::{Parser, Subcommand};
 use dump::{DumpWriter, IndexMetadata};
 use file_store::FileStore;
@@ -10,9 +10,10 @@ use meilisearch_auth::AuthController;
 use meilisearch_types::heed::types::{SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
+use meilisearch_types::milli::index::{db_name, main_key};
 use meilisearch_types::milli::{obkv_to_json, BEU32};
 use meilisearch_types::tasks::{Status, Task};
-use meilisearch_types::versioning::check_version_file;
+use meilisearch_types::versioning::{create_version_file, get_version, parse_version};
 use meilisearch_types::Index;
 use time::macros::format_description;
 use time::OffsetDateTime;
@@ -62,21 +63,458 @@ enum Command {
         #[arg(long)]
         skip_enqueued_tasks: bool,
     },
+
+    /// Attempts to upgrade from one major version to the next without a dump.
+    ///
+    /// Make sure to run this commmand when Meilisearch is not running!
+    /// If Meilisearch is running while executing this command, the database could be corrupted
+    /// (contain data from both the old and the new versions)
+    ///
+    /// Supported upgrade paths:
+    ///
+    /// - v1.9.0 -> v1.10.0
+    OfflineUpgrade {
+        #[arg(long)]
+        target_version: String,
+    },
 }
 
 fn main() -> anyhow::Result<()> {
     let Cli { db_path, command } = Cli::parse();
 
-    check_version_file(&db_path).context("While checking the version file")?;
+    let detected_version = get_version(&db_path).context("While checking the version file")?;
 
     match command {
         Command::ClearTaskQueue => clear_task_queue(db_path),
         Command::ExportADump { dump_dir, skip_enqueued_tasks } => {
             export_a_dump(db_path, dump_dir, skip_enqueued_tasks)
         }
+        Command::OfflineUpgrade { target_version } => {
+            let target_version = parse_version(&target_version).context("While parsing `--target-version`. Make sure `--target-version` is in the format MAJOR.MINOR.PATCH")?;
+            OfflineUpgrade { db_path, current_version: detected_version, target_version }.upgrade()
+        }
     }
 }
 
+struct OfflineUpgrade {
+    db_path: PathBuf,
+    current_version: (String, String, String),
+    target_version: (String, String, String),
+}
+
+impl OfflineUpgrade {
+    fn upgrade(self) -> anyhow::Result<()> {
+        // TODO: if we make this process support more versions, introduce a more flexible way of checking for the version
+        // currently only supports v1.9 to v1.10
+        let (current_major, current_minor, current_patch) = &self.current_version;
+
+        match (current_major.as_str(), current_minor.as_str(), current_patch.as_str()) {
+            ("1", "9", _) => {}
+            _ => {
+                bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9")
+            }
+        }
+
+        let (target_major, target_minor, target_patch) = &self.target_version;
+
+        match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) {
+            ("1", "10", _) => {}
+            _ => {
+                bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10")
+            }
+        }
+
+        println!("Upgrading from {current_major}.{current_minor}.{current_patch} to {target_major}.{target_minor}.{target_patch}");
+
+        self.v1_9_to_v1_10()?;
+
+        println!("Writing VERSION file");
+
+        create_version_file(&self.db_path, target_major, target_minor, target_patch)
+            .context("while writing VERSION file after the upgrade")?;
+
+        println!("Success");
+
+        Ok(())
+    }
+
+    fn v1_9_to_v1_10(&self) -> anyhow::Result<()> {
+        // 2 changes here
+
+        // 1. date format. needs to be done before opening the Index
+        // 2. REST embedders. We don't support this case right now, so bail
+
+        let index_scheduler_path = self.db_path.join("tasks");
+        let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) }
+            .with_context(|| {
+                format!("While trying to open {:?}", index_scheduler_path.display())
+            })?;
+
+        let mut sched_wtxn = env.write_txn()?;
+
+        let index_mapping: Database<Str, UuidCodec> =
+            try_opening_database(&env, &sched_wtxn, "index-mapping")?;
+
+        let index_stats: Database<UuidCodec, Unspecified> =
+            try_opening_database(&env, &sched_wtxn, "index-stats").with_context(|| {
+                format!("While trying to open {:?}", index_scheduler_path.display())
+            })?;
+
+        let index_count =
+            index_mapping.len(&sched_wtxn).context("while reading the number of indexes")?;
+
+        // FIXME: not ideal, we have to pre-populate all indexes to prevent double borrow of sched_wtxn
+        // 1. immutably for the iteration
+        // 2. mutably for updating index stats
+        let indexes: Vec<_> = index_mapping
+            .iter(&sched_wtxn)?
+            .map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid)))
+            .collect();
+
+        let mut rest_embedders = Vec::new();
+
+        let mut unwrapped_indexes = Vec::new();
+
+        // check that update can take place
+        for (index_index, result) in indexes.into_iter().enumerate() {
+            let (uid, uuid) = result?;
+            let index_path = self.db_path.join("indexes").join(uuid.to_string());
+
+            println!(
+                "[{}/{index_count}]Checking that update can take place for  `{uid}` at `{}`",
+                index_index + 1,
+                index_path.display()
+            );
+
+            let index_env = unsafe {
+                // FIXME: fetch the 25 magic number from the index file
+                EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
+                    format!("while opening index {uid} at '{}'", index_path.display())
+                })?
+            };
+
+            let index_txn = index_env.read_txn().with_context(|| {
+                format!(
+                    "while obtaining a write transaction for index {uid} at {}",
+                    index_path.display()
+                )
+            })?;
+
+            println!("\t- Checking for incompatible embedders (REST embedders)");
+            let rest_embedders_for_index = find_rest_embedders(&uid, &index_env, &index_txn)?;
+
+            if rest_embedders_for_index.is_empty() {
+                unwrapped_indexes.push((uid, uuid));
+            } else {
+                // no need to add to unwrapped indexes because we'll exit early
+                rest_embedders.push((uid, rest_embedders_for_index));
+            }
+        }
+
+        if !rest_embedders.is_empty() {
+            let rest_embedders = rest_embedders
+                .into_iter()
+                .flat_map(|(index, embedders)| std::iter::repeat(index.clone()).zip(embedders))
+                .map(|(index, embedder)| format!("\t- embedder `{embedder}` in index `{index}`"))
+                .collect::<Vec<_>>()
+                .join("\n");
+            bail!("The update cannot take place because there are REST embedder(s). Remove them before proceeding with the update:\n{rest_embedders}\n\n\
+            The database has not been modified and is still a valid v1.9 database.");
+        }
+
+        println!("Update can take place, updating");
+
+        for (index_index, (uid, uuid)) in unwrapped_indexes.into_iter().enumerate() {
+            let index_path = self.db_path.join("indexes").join(uuid.to_string());
+
+            println!(
+                "[{}/{index_count}]Updating index `{uid}` at `{}`",
+                index_index + 1,
+                index_path.display()
+            );
+
+            let index_env = unsafe {
+                // FIXME: fetch the 25 magic number from the index file
+                EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
+                    format!("while opening index {uid} at '{}'", index_path.display())
+                })?
+            };
+
+            let mut index_wtxn = index_env.write_txn().with_context(|| {
+                format!(
+                    "while obtaining a write transaction for index `{uid}` at `{}`",
+                    index_path.display()
+                )
+            })?;
+
+            println!("\t- Updating index stats");
+            update_index_stats(index_stats, &uid, uuid, &mut sched_wtxn)?;
+            println!("\t- Updating date format");
+            update_date_format(&uid, &index_env, &mut index_wtxn)?;
+
+            index_wtxn.commit().with_context(|| {
+                format!(
+                    "while committing the write txn for index `{uid}` at {}",
+                    index_path.display()
+                )
+            })?;
+        }
+
+        sched_wtxn.commit().context("while committing the write txn for the index-scheduler")?;
+
+        println!("Upgrading database succeeded");
+
+        Ok(())
+    }
+}
+
+pub mod v1_9 {
+    pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
+
+    /// The statistics that can be computed from an `Index` object.
+    #[derive(serde::Serialize, serde::Deserialize, Debug)]
+    pub struct IndexStats {
+        /// Number of documents in the index.
+        pub number_of_documents: u64,
+        /// Size taken up by the index' DB, in bytes.
+        ///
+        /// This includes the size taken by both the used and free pages of the DB, and as the free pages
+        /// are not returned to the disk after a deletion, this number is typically larger than
+        /// `used_database_size` that only includes the size of the used pages.
+        pub database_size: u64,
+        /// Size taken by the used pages of the index' DB, in bytes.
+        ///
+        /// As the DB backend does not return to the disk the pages that are not currently used by the DB,
+        /// this value is typically smaller than `database_size`.
+        pub used_database_size: u64,
+        /// Association of every field name with the number of times it occurs in the documents.
+        pub field_distribution: FieldDistribution,
+        /// Creation date of the index.
+        pub created_at: time::OffsetDateTime,
+        /// Date of the last update of the index.
+        pub updated_at: time::OffsetDateTime,
+    }
+
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Debug, Deserialize, Serialize)]
+    pub struct IndexEmbeddingConfig {
+        pub name: String,
+        pub config: EmbeddingConfig,
+    }
+
+    #[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
+    pub struct EmbeddingConfig {
+        /// Options of the embedder, specific to each kind of embedder
+        pub embedder_options: EmbedderOptions,
+    }
+
+    /// Options of an embedder, specific to each kind of embedder.
+    #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+    pub enum EmbedderOptions {
+        HuggingFace(hf::EmbedderOptions),
+        OpenAi(openai::EmbedderOptions),
+        Ollama(ollama::EmbedderOptions),
+        UserProvided(manual::EmbedderOptions),
+        Rest(rest::EmbedderOptions),
+    }
+
+    impl Default for EmbedderOptions {
+        fn default() -> Self {
+            Self::OpenAi(openai::EmbedderOptions { api_key: None, dimensions: None })
+        }
+    }
+
+    mod hf {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub model: String,
+            pub revision: Option<String>,
+        }
+    }
+    mod openai {
+
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub api_key: Option<String>,
+            pub dimensions: Option<usize>,
+        }
+    }
+    mod ollama {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub embedding_model: String,
+            pub url: Option<String>,
+            pub api_key: Option<String>,
+        }
+    }
+    mod manual {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub dimensions: usize,
+        }
+    }
+    mod rest {
+        #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize, Hash)]
+        pub struct EmbedderOptions {
+            pub api_key: Option<String>,
+            pub dimensions: Option<usize>,
+            pub url: String,
+            pub input_field: Vec<String>,
+            // path to the array of embeddings
+            pub path_to_embeddings: Vec<String>,
+            // shape of a single embedding
+            pub embedding_object: Vec<String>,
+        }
+    }
+
+    pub type OffsetDateTime = time::OffsetDateTime;
+}
+
+pub mod v1_10 {
+    use crate::v1_9;
+
+    pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
+
+    /// The statistics that can be computed from an `Index` object.
+    #[derive(serde::Serialize, serde::Deserialize, Debug)]
+    pub struct IndexStats {
+        /// Number of documents in the index.
+        pub number_of_documents: u64,
+        /// Size taken up by the index' DB, in bytes.
+        ///
+        /// This includes the size taken by both the used and free pages of the DB, and as the free pages
+        /// are not returned to the disk after a deletion, this number is typically larger than
+        /// `used_database_size` that only includes the size of the used pages.
+        pub database_size: u64,
+        /// Size taken by the used pages of the index' DB, in bytes.
+        ///
+        /// As the DB backend does not return to the disk the pages that are not currently used by the DB,
+        /// this value is typically smaller than `database_size`.
+        pub used_database_size: u64,
+        /// Association of every field name with the number of times it occurs in the documents.
+        pub field_distribution: FieldDistribution,
+        /// Creation date of the index.
+        #[serde(with = "time::serde::rfc3339")]
+        pub created_at: time::OffsetDateTime,
+        /// Date of the last update of the index.
+        #[serde(with = "time::serde::rfc3339")]
+        pub updated_at: time::OffsetDateTime,
+    }
+
+    impl From<v1_9::IndexStats> for IndexStats {
+        fn from(
+            v1_9::IndexStats {
+                number_of_documents,
+                database_size,
+                used_database_size,
+                field_distribution,
+                created_at,
+                updated_at,
+            }: v1_9::IndexStats,
+        ) -> Self {
+            IndexStats {
+                number_of_documents,
+                database_size,
+                used_database_size,
+                field_distribution,
+                created_at,
+                updated_at,
+            }
+        }
+    }
+
+    #[derive(serde::Serialize, serde::Deserialize)]
+    #[serde(transparent)]
+    pub struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime);
+}
+
+fn update_index_stats(
+    index_stats: Database<UuidCodec, Unspecified>,
+    index_uid: &str,
+    index_uuid: uuid::Uuid,
+    sched_wtxn: &mut RwTxn,
+) -> anyhow::Result<()> {
+    let ctx = || format!("while updating index stats for index `{index_uid}`");
+
+    let stats: Option<v1_9::IndexStats> = index_stats
+        .remap_data_type::<SerdeJson<v1_9::IndexStats>>()
+        .get(sched_wtxn, &index_uuid)
+        .with_context(ctx)?;
+
+    if let Some(stats) = stats {
+        let stats: v1_10::IndexStats = stats.into();
+
+        index_stats
+            .remap_data_type::<SerdeJson<v1_10::IndexStats>>()
+            .put(sched_wtxn, &index_uuid, &stats)
+            .with_context(ctx)?;
+    }
+
+    Ok(())
+}
+
+fn update_date_format(
+    index_uid: &str,
+    index_env: &Env,
+    index_wtxn: &mut RwTxn,
+) -> anyhow::Result<()> {
+    let main = try_opening_poly_database(index_env, index_wtxn, db_name::MAIN)
+        .with_context(|| format!("while updating date format for index `{index_uid}`"))?;
+
+    date_round_trip(index_wtxn, index_uid, main, main_key::CREATED_AT_KEY)?;
+    date_round_trip(index_wtxn, index_uid, main, main_key::UPDATED_AT_KEY)?;
+
+    Ok(())
+}
+
+fn find_rest_embedders(
+    index_uid: &str,
+    index_env: &Env,
+    index_txn: &RoTxn,
+) -> anyhow::Result<Vec<String>> {
+    let main = try_opening_poly_database(index_env, index_txn, db_name::MAIN)
+        .with_context(|| format!("while checking REST embedders for index `{index_uid}`"))?;
+
+    let mut rest_embedders = vec![];
+
+    for config in main
+        .remap_types::<Str, SerdeJson<Vec<v1_9::IndexEmbeddingConfig>>>()
+        .get(index_txn, main_key::EMBEDDING_CONFIGS)?
+        .unwrap_or_default()
+    {
+        if let v1_9::EmbedderOptions::Rest(_) = config.config.embedder_options {
+            rest_embedders.push(config.name);
+        }
+    }
+
+    Ok(rest_embedders)
+}
+
+fn date_round_trip(
+    wtxn: &mut RwTxn,
+    index_uid: &str,
+    db: Database<Unspecified, Unspecified>,
+    key: &str,
+) -> anyhow::Result<()> {
+    let datetime =
+        db.remap_types::<Str, SerdeJson<v1_9::OffsetDateTime>>().get(wtxn, key).with_context(
+            || format!("could not read `{key}` while updating date format for index `{index_uid}`"),
+        )?;
+
+    if let Some(datetime) = datetime {
+        db.remap_types::<Str, SerdeJson<v1_10::OffsetDateTime>>()
+            .put(wtxn, key, &v1_10::OffsetDateTime(datetime))
+            .with_context(|| {
+                format!(
+                    "could not write `{key}` while updating date format for index `{index_uid}`"
+                )
+            })?;
+    }
+
+    Ok(())
+}
+
 /// Clears the task queue located at `db_path`.
 fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> {
     let path = db_path.join("tasks");

milli/src/error.rs

@@ -258,6 +258,8 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
     },
     #[error("`.embedders.{embedder_name}.dimensions`: `dimensions` cannot be zero")]
     InvalidSettingsDimensions { embedder_name: String },
+    #[error("`.embedders.{embedder_name}.documentTemplateMaxBytes`: `documentTemplateMaxBytes` cannot be zero")]
+    InvalidSettingsDocumentTemplateMaxBytes { embedder_name: String },
     #[error("`.embedders.{embedder_name}.url`: could not parse `{url}`: {inner_error}")]
     InvalidUrl { embedder_name: String, inner_error: url::ParseError, url: String },
     #[error("Document editions cannot modify a document's primary key")]

milli/src/prompt/context.rs

@@ -5,7 +5,7 @@ use liquid::{ObjectView, ValueView};
 
 use super::document::Document;
 use super::fields::Fields;
-use crate::FieldsIdsMap;
+use super::FieldsIdsMapWithMetadata;
 
 #[derive(Debug, Clone)]
 pub struct Context<'a> {
@@ -14,7 +14,7 @@ pub struct Context<'a> {
 }
 
 impl<'a> Context<'a> {
-    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMap) -> Self {
+    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMapWithMetadata<'a>) -> Self {
         Self { document, fields: Fields::new(document, field_id_map) }
     }
 }

milli/src/prompt/fields.rs

@@ -4,16 +4,20 @@ use liquid::model::{
 use liquid::{ObjectView, ValueView};
 
 use super::document::Document;
-use crate::FieldsIdsMap;
+use super::{FieldMetadata, FieldsIdsMapWithMetadata};
 #[derive(Debug, Clone)]
 pub struct Fields<'a>(Vec<FieldValue<'a>>);
 
 impl<'a> Fields<'a> {
-    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMap) -> Self {
+    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMapWithMetadata<'a>) -> Self {
         Self(
             std::iter::repeat(document)
                 .zip(field_id_map.iter())
-                .map(|(document, (_fid, name))| FieldValue { document, name })
+                .map(|(document, (fid, name))| FieldValue {
+                    document,
+                    name,
+                    metadata: field_id_map.metadata(fid).unwrap_or_default(),
+                })
                 .collect(),
         )
     }
@@ -23,6 +27,7 @@ impl<'a> Fields<'a> {
 pub struct FieldValue<'a> {
     name: &'a str,
     document: &'a Document<'a>,
+    metadata: FieldMetadata,
 }
 
 impl<'a> ValueView for FieldValue<'a> {
@@ -74,6 +79,10 @@ impl<'a> FieldValue<'a> {
         self.document.get(self.name).unwrap_or(&LiquidValue::Nil)
     }
 
+    pub fn is_searchable(&self) -> &bool {
+        &self.metadata.searchable
+    }
+
     pub fn is_empty(&self) -> bool {
         self.size() == 0
     }
@@ -89,12 +98,14 @@ impl<'a> ObjectView for FieldValue<'a> {
     }
 
     fn keys<'k>(&'k self) -> Box<dyn Iterator<Item = KStringCow<'k>> + 'k> {
-        Box::new(["name", "value"].iter().map(|&x| KStringCow::from_static(x)))
+        Box::new(["name", "value", "is_searchable"].iter().map(|&x| KStringCow::from_static(x)))
     }
 
     fn values<'k>(&'k self) -> Box<dyn Iterator<Item = &'k dyn ValueView> + 'k> {
         Box::new(
-            std::iter::once(self.name() as &dyn ValueView).chain(std::iter::once(self.value())),
+            std::iter::once(self.name() as &dyn ValueView)
+                .chain(std::iter::once(self.value()))
+                .chain(std::iter::once(self.is_searchable() as &dyn ValueView)),
         )
     }
 
@@ -103,13 +114,14 @@ impl<'a> ObjectView for FieldValue<'a> {
     }
 
     fn contains_key(&self, index: &str) -> bool {
-        index == "name" || index == "value"
+        index == "name" || index == "value" || index == "is_searchable"
     }
 
     fn get<'s>(&'s self, index: &str) -> Option<&'s dyn ValueView> {
         match index {
             "name" => Some(self.name()),
             "value" => Some(self.value()),
+            "is_searchable" => Some(self.is_searchable()),
             _ => None,
         }
     }

milli/src/prompt/mod.rs

@@ -4,28 +4,33 @@ pub(crate) mod error;
 mod fields;
 mod template_checker;
 
+use std::collections::BTreeMap;
 use std::convert::TryFrom;
+use std::num::NonZeroUsize;
+use std::ops::Deref;
 
 use error::{NewPromptError, RenderPromptError};
 
 use self::context::Context;
 use self::document::Document;
 use crate::update::del_add::DelAdd;
-use crate::FieldsIdsMap;
+use crate::{FieldId, FieldsIdsMap};
 
 pub struct Prompt {
     template: liquid::Template,
     template_text: String,
+    max_bytes: Option<NonZeroUsize>,
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct PromptData {
     pub template: String,
+    pub max_bytes: Option<NonZeroUsize>,
 }
 
 impl From<Prompt> for PromptData {
     fn from(value: Prompt) -> Self {
-        Self { template: value.template_text }
+        Self { template: value.template_text, max_bytes: value.max_bytes }
     }
 }
 
@@ -33,14 +38,18 @@ impl TryFrom<PromptData> for Prompt {
     type Error = NewPromptError;
 
     fn try_from(value: PromptData) -> Result<Self, Self::Error> {
-        Prompt::new(value.template)
+        Prompt::new(value.template, value.max_bytes)
     }
 }
 
 impl Clone for Prompt {
     fn clone(&self) -> Self {
         let template_text = self.template_text.clone();
-        Self { template: new_template(&template_text).unwrap(), template_text }
+        Self {
+            template: new_template(&template_text).unwrap(),
+            template_text,
+            max_bytes: self.max_bytes,
+        }
     }
 }
 
@@ -53,25 +62,35 @@ fn default_template() -> liquid::Template {
 }
 
 fn default_template_text() -> &'static str {
-    "{% for field in fields %} \
+    "{% for field in fields %}\
+    {% if field.is_searchable and field.value != nil %}\
     {{ field.name }}: {{ field.value }}\n\
+    {% endif %}\
     {% endfor %}"
 }
 
+pub fn default_max_bytes() -> NonZeroUsize {
+    NonZeroUsize::new(400).unwrap()
+}
+
 impl Default for Prompt {
     fn default() -> Self {
-        Self { template: default_template(), template_text: default_template_text().into() }
+        Self {
+            template: default_template(),
+            template_text: default_template_text().into(),
+            max_bytes: Some(default_max_bytes()),
+        }
     }
 }
 
 impl Default for PromptData {
     fn default() -> Self {
-        Self { template: default_template_text().into() }
+        Self { template: default_template_text().into(), max_bytes: Some(default_max_bytes()) }
     }
 }
 
 impl Prompt {
-    pub fn new(template: String) -> Result<Self, NewPromptError> {
+    pub fn new(template: String, max_bytes: Option<NonZeroUsize>) -> Result<Self, NewPromptError> {
         let this = Self {
             template: liquid::ParserBuilder::with_stdlib()
                 .build()
@@ -79,6 +98,7 @@ impl Prompt {
                 .parse(&template)
                 .map_err(NewPromptError::cannot_parse_template)?,
             template_text: template,
+            max_bytes,
         };
 
         // render template with special object that's OK with `doc.*` and `fields.*`
@@ -93,20 +113,72 @@ impl Prompt {
         &self,
         document: obkv::KvReaderU16<'_>,
         side: DelAdd,
-        field_id_map: &FieldsIdsMap,
+        field_id_map: &FieldsIdsMapWithMetadata,
     ) -> Result<String, RenderPromptError> {
         let document = Document::new(document, side, field_id_map);
         let context = Context::new(&document, field_id_map);
 
-        self.template.render(&context).map_err(RenderPromptError::missing_context)
+        let mut rendered =
+            self.template.render(&context).map_err(RenderPromptError::missing_context)?;
+        if let Some(max_bytes) = self.max_bytes {
+            truncate(&mut rendered, max_bytes.get());
+        }
+        Ok(rendered)
     }
 }
 
+fn truncate(s: &mut String, max_bytes: usize) {
+    if max_bytes >= s.len() {
+        return;
+    }
+    for i in (0..=max_bytes).rev() {
+        if s.is_char_boundary(i) {
+            s.truncate(i);
+            break;
+        }
+    }
+}
+
+pub struct FieldsIdsMapWithMetadata<'a> {
+    fields_ids_map: &'a FieldsIdsMap,
+    metadata: BTreeMap<FieldId, FieldMetadata>,
+}
+
+impl<'a> FieldsIdsMapWithMetadata<'a> {
+    pub fn new(fields_ids_map: &'a FieldsIdsMap, searchable_fields_ids: &'_ [FieldId]) -> Self {
+        let mut metadata: BTreeMap<FieldId, FieldMetadata> =
+            fields_ids_map.ids().map(|id| (id, Default::default())).collect();
+        for searchable_field_id in searchable_fields_ids {
+            let Some(metadata) = metadata.get_mut(searchable_field_id) else { continue };
+            metadata.searchable = true;
+        }
+        Self { fields_ids_map, metadata }
+    }
+
+    pub fn metadata(&self, field_id: FieldId) -> Option<FieldMetadata> {
+        self.metadata.get(&field_id).copied()
+    }
+}
+
+impl<'a> Deref for FieldsIdsMapWithMetadata<'a> {
+    type Target = FieldsIdsMap;
+
+    fn deref(&self) -> &Self::Target {
+        self.fields_ids_map
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy)]
+pub struct FieldMetadata {
+    pub searchable: bool,
+}
+
 #[cfg(test)]
 mod test {
     use super::Prompt;
     use crate::error::FaultSource;
     use crate::prompt::error::{NewPromptError, NewPromptErrorKind};
+    use crate::prompt::truncate;
 
     #[test]
     fn default_template() {
@@ -116,18 +188,18 @@ mod test {
 
     #[test]
     fn empty_template() {
-        Prompt::new("".into()).unwrap();
+        Prompt::new("".into(), None).unwrap();
     }
 
     #[test]
     fn template_ok() {
-        Prompt::new("{{doc.title}}: {{doc.overview}}".into()).unwrap();
+        Prompt::new("{{doc.title}}: {{doc.overview}}".into(), None).unwrap();
     }
 
     #[test]
     fn template_syntax() {
         assert!(matches!(
-            Prompt::new("{{doc.title: {{doc.overview}}".into()),
+            Prompt::new("{{doc.title: {{doc.overview}}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::CannotParseTemplate(_),
                 fault: FaultSource::User
@@ -138,7 +210,7 @@ mod test {
     #[test]
     fn template_missing_doc() {
         assert!(matches!(
-            Prompt::new("{{title}}: {{overview}}".into()),
+            Prompt::new("{{title}}: {{overview}}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::InvalidFieldsInTemplate(_),
                 fault: FaultSource::User
@@ -148,29 +220,62 @@ mod test {
 
     #[test]
     fn template_nested_doc() {
-        Prompt::new("{{doc.actor.firstName}}: {{doc.actor.lastName}}".into()).unwrap();
+        Prompt::new("{{doc.actor.firstName}}: {{doc.actor.lastName}}".into(), None).unwrap();
     }
 
     #[test]
     fn template_fields() {
-        Prompt::new("{% for field in fields %}{{field}}{% endfor %}".into()).unwrap();
+        Prompt::new("{% for field in fields %}{{field}}{% endfor %}".into(), None).unwrap();
     }
 
     #[test]
     fn template_fields_ok() {
-        Prompt::new("{% for field in fields %}{{field.name}}: {{field.value}}{% endfor %}".into())
-            .unwrap();
+        Prompt::new(
+            "{% for field in fields %}{{field.name}}: {{field.value}}{% endfor %}".into(),
+            None,
+        )
+        .unwrap();
     }
 
     #[test]
     fn template_fields_invalid() {
         assert!(matches!(
             // intentionally garbled field
-            Prompt::new("{% for field in fields %}{{field.vaelu}} {% endfor %}".into()),
+            Prompt::new("{% for field in fields %}{{field.vaelu}} {% endfor %}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::InvalidFieldsInTemplate(_),
                 fault: FaultSource::User
             })
         ));
     }
+
+    // todo: test truncation
+    #[test]
+    fn template_truncation() {
+        let mut s = "インテル ザー ビーグル".to_string();
+
+        truncate(&mut s, 42);
+        assert_eq!(s, "インテル ザー ビーグル");
+
+        assert_eq!(s.len(), 32);
+        truncate(&mut s, 32);
+        assert_eq!(s, "インテル ザー ビーグル");
+
+        truncate(&mut s, 31);
+        assert_eq!(s, "インテル ザー ビーグ");
+        truncate(&mut s, 30);
+        assert_eq!(s, "インテル ザー ビーグ");
+        truncate(&mut s, 28);
+        assert_eq!(s, "インテル ザー ビー");
+        truncate(&mut s, 26);
+        assert_eq!(s, "インテル ザー ビー");
+        truncate(&mut s, 25);
+        assert_eq!(s, "インテル ザー ビ");
+
+        assert_eq!("イ".len(), 3);
+        truncate(&mut s, 3);
+        assert_eq!(s, "イ");
+        truncate(&mut s, 2);
+        assert_eq!(s, "");
+    }
 }

milli/src/search/facet/filter.rs

@@ -75,25 +75,21 @@ impl<'a> Display for FilterError<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::AttributeNotFilterable { attribute, filterable_fields } => {
+                write!(f, "Attribute `{attribute}` is not filterable.")?;
                 if filterable_fields.is_empty() {
-                    write!(
-                        f,
-                        "Attribute `{}` is not filterable. This index does not have configured filterable attributes.",
-                        attribute,
-                    )
+                    write!(f, " This index does not have configured filterable attributes.")
                 } else {
-                    let filterables_list = filterable_fields
-                        .iter()
-                        .map(AsRef::as_ref)
-                        .collect::<Vec<&str>>()
-                        .join(" ");
-
-                    write!(
-                        f,
-                        "Attribute `{}` is not filterable. Available filterable attributes are: `{}`.",
-                        attribute,
-                        filterables_list,
-                    )
+                    write!(f, " Available filterable attributes are: ")?;
+                    let mut filterables_list =
+                        filterable_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>();
+                    filterables_list.sort_unstable();
+                    for (idx, filterable) in filterables_list.iter().enumerate() {
+                        write!(f, "`{filterable}`")?;
+                        if idx != filterables_list.len() - 1 {
+                            write!(f, ", ")?;
+                        }
+                    }
+                    write!(f, ".")
                 }
             }
             Self::TooDeep => write!(

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -15,14 +15,14 @@ use serde_json::Value;
 use super::helpers::{create_writer, writer_into_reader, GrenadParameters};
 use crate::error::FaultSource;
 use crate::index::IndexEmbeddingConfig;
-use crate::prompt::Prompt;
+use crate::prompt::{FieldsIdsMapWithMetadata, Prompt};
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::settings::InnerIndexSettingsDiff;
 use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution};
 use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState, RESERVED_VECTORS_FIELD_NAME};
 use crate::vector::settings::{EmbedderAction, ReindexAction};
 use crate::vector::{Embedder, Embeddings};
-use crate::{try_split_array_at, DocumentId, FieldId, FieldsIdsMap, Result, ThreadPoolNoAbort};
+use crate::{try_split_array_at, DocumentId, FieldId, Result, ThreadPoolNoAbort};
 
 /// The length of the elements that are always in the buffer when inserting new values.
 const TRUNCATE_SIZE: usize = size_of::<DocumentId>();
@@ -189,7 +189,13 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     let reindex_vectors = settings_diff.reindex_vectors();
 
     let old_fields_ids_map = &settings_diff.old.fields_ids_map;
+    let old_fields_ids_map =
+        FieldsIdsMapWithMetadata::new(old_fields_ids_map, &settings_diff.old.searchable_fields_ids);
+
     let new_fields_ids_map = &settings_diff.new.fields_ids_map;
+    let new_fields_ids_map =
+        FieldsIdsMapWithMetadata::new(new_fields_ids_map, &settings_diff.new.searchable_fields_ids);
+
     // the vector field id may have changed
     let old_vectors_fid = old_fields_ids_map.id(RESERVED_VECTORS_FIELD_NAME);
 
@@ -376,7 +382,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                             );
                             continue;
                         }
-                        regenerate_prompt(obkv, prompt, new_fields_ids_map)?
+                        regenerate_prompt(obkv, prompt, &new_fields_ids_map)?
                     }
                 },
                 // prompt regeneration is only triggered for existing embedders
@@ -393,7 +399,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                         regenerate_if_prompt_changed(
                             obkv,
                             (old_prompt, prompt),
-                            (old_fields_ids_map, new_fields_ids_map),
+                            (&old_fields_ids_map, &new_fields_ids_map),
                         )?
                     } else {
                         // we can simply ignore user provided vectors as they are not regenerated and are
@@ -409,7 +415,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                     prompt,
                     (add_to_user_provided, remove_from_user_provided),
                     (old, new),
-                    (old_fields_ids_map, new_fields_ids_map),
+                    (&old_fields_ids_map, &new_fields_ids_map),
                     document_id,
                     embedder_name,
                     embedder_is_manual,
@@ -479,7 +485,10 @@ fn extract_vector_document_diff(
     prompt: &Prompt,
     (add_to_user_provided, remove_from_user_provided): (&mut RoaringBitmap, &mut RoaringBitmap),
     (old, new): (VectorState, VectorState),
-    (old_fields_ids_map, new_fields_ids_map): (&FieldsIdsMap, &FieldsIdsMap),
+    (old_fields_ids_map, new_fields_ids_map): (
+        &FieldsIdsMapWithMetadata,
+        &FieldsIdsMapWithMetadata,
+    ),
     document_id: impl Fn() -> Value,
     embedder_name: &str,
     embedder_is_manual: bool,
@@ -599,7 +608,10 @@ fn extract_vector_document_diff(
 fn regenerate_if_prompt_changed(
     obkv: obkv::KvReader<'_, FieldId>,
     (old_prompt, new_prompt): (&Prompt, &Prompt),
-    (old_fields_ids_map, new_fields_ids_map): (&FieldsIdsMap, &FieldsIdsMap),
+    (old_fields_ids_map, new_fields_ids_map): (
+        &FieldsIdsMapWithMetadata,
+        &FieldsIdsMapWithMetadata,
+    ),
 ) -> Result<VectorStateDelta> {
     let old_prompt =
         old_prompt.render(obkv, DelAdd::Deletion, old_fields_ids_map).unwrap_or(Default::default());
@@ -614,7 +626,7 @@ fn regenerate_if_prompt_changed(
 fn regenerate_prompt(
     obkv: obkv::KvReader<'_, FieldId>,
     prompt: &Prompt,
-    new_fields_ids_map: &FieldsIdsMap,
+    new_fields_ids_map: &FieldsIdsMapWithMetadata,
 ) -> Result<VectorStateDelta> {
     let prompt = prompt.render(obkv, DelAdd::Addition, new_fields_ids_map)?;
 

milli/src/update/index_documents/mod.rs

@@ -2740,6 +2740,7 @@ mod tests {
                         api_key: Setting::NotSet,
                         dimensions: Setting::Set(3),
                         document_template: Setting::NotSet,
+                        document_template_max_bytes: Setting::NotSet,
                         url: Setting::NotSet,
                         request: Setting::NotSet,
                         response: Setting::NotSet,

milli/src/update/settings.rs

@@ -1,5 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::convert::TryInto;
+use std::num::NonZeroUsize;
 use std::result::Result as StdResult;
 use std::sync::Arc;
 
@@ -19,6 +20,7 @@ use crate::index::{
     IndexEmbeddingConfig, DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS,
 };
 use crate::order_by_map::OrderByMap;
+use crate::prompt::default_max_bytes;
 use crate::proximity::ProximityPrecision;
 use crate::update::index_documents::IndexDocumentsMethod;
 use crate::update::{IndexDocuments, UpdateIndexingStep};
@@ -1238,7 +1240,7 @@ impl InnerIndexSettingsDiff {
         old_settings: InnerIndexSettings,
         new_settings: InnerIndexSettings,
         primary_key_id: Option<FieldId>,
-        embedding_config_updates: BTreeMap<String, EmbedderAction>,
+        mut embedding_config_updates: BTreeMap<String, EmbedderAction>,
         settings_update_only: bool,
     ) -> Self {
         let only_additional_fields = match (
@@ -1273,6 +1275,32 @@ impl InnerIndexSettingsDiff {
         let cache_user_defined_searchables = old_settings.user_defined_searchable_fields
             != new_settings.user_defined_searchable_fields;
 
+        // if the user-defined searchables changed, then we need to reindex prompts.
+        if cache_user_defined_searchables {
+            for (embedder_name, (config, _)) in new_settings.embedding_configs.inner_as_ref() {
+                // skip embedders that don't use document templates
+                if !config.uses_document_template() {
+                    continue;
+                }
+
+                // note: this could currently be entry.or_insert(..), but we're future-proofing with an explicit match
+                // this always makes the code clearer by explicitly handling the cases
+                match embedding_config_updates.entry(embedder_name.clone()) {
+                    std::collections::btree_map::Entry::Vacant(entry) => {
+                        entry.insert(EmbedderAction::Reindex(ReindexAction::RegeneratePrompts));
+                    }
+                    std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
+                        EmbedderAction::WriteBackToDocuments(_) => { /* we are deleting this embedder, so no point in regeneration */
+                        }
+                        EmbedderAction::Reindex(ReindexAction::FullReindex) => { /* we are already fully reindexing */
+                        }
+                        EmbedderAction::Reindex(ReindexAction::RegeneratePrompts) => { /* we are already regenerating prompts */
+                        }
+                    },
+                };
+            }
+        }
+
         InnerIndexSettingsDiff {
             old: old_settings,
             new: new_settings,
@@ -1547,16 +1575,30 @@ fn validate_prompt(
             api_key,
             dimensions,
             document_template: Setting::Set(template),
+            document_template_max_bytes,
             url,
             request,
             response,
             distribution,
             headers,
         }) => {
+            let max_bytes = match document_template_max_bytes.set() {
+                Some(max_bytes) => NonZeroUsize::new(max_bytes).ok_or_else(|| {
+                    crate::error::UserError::InvalidSettingsDocumentTemplateMaxBytes {
+                        embedder_name: name.to_owned(),
+                    }
+                })?,
+                None => default_max_bytes(),
+            };
+
             // validate
-            let template = crate::prompt::Prompt::new(template)
-                .map(|prompt| crate::prompt::PromptData::from(prompt).template)
-                .map_err(|inner| UserError::InvalidPromptForEmbeddings(name.to_owned(), inner))?;
+            let template = crate::prompt::Prompt::new(
+                template,
+                // always specify a max_bytes
+                Some(max_bytes),
+            )
+            .map(|prompt| crate::prompt::PromptData::from(prompt).template)
+            .map_err(|inner| UserError::InvalidPromptForEmbeddings(name.to_owned(), inner))?;
 
             Ok(Setting::Set(EmbeddingSettings {
                 source,
@@ -1565,6 +1607,7 @@ fn validate_prompt(
                 api_key,
                 dimensions,
                 document_template: Setting::Set(template),
+                document_template_max_bytes,
                 url,
                 request,
                 response,
@@ -1589,6 +1632,7 @@ pub fn validate_embedding_settings(
         api_key,
         dimensions,
         document_template,
+        document_template_max_bytes,
         url,
         request,
         response,
@@ -1628,6 +1672,7 @@ pub fn validate_embedding_settings(
             api_key,
             dimensions,
             document_template,
+            document_template_max_bytes,
             url,
             request,
             response,
@@ -1700,6 +1745,12 @@ pub fn validate_embedding_settings(
                 inferred_source,
                 name,
             )?;
+            check_unset(
+                &document_template_max_bytes,
+                EmbeddingSettings::DOCUMENT_TEMPLATE_MAX_BYTES,
+                inferred_source,
+                name,
+            )?;
             check_set(&dimensions, EmbeddingSettings::DIMENSIONS, inferred_source, name)?;
 
             check_unset(&url, EmbeddingSettings::URL, inferred_source, name)?;
@@ -1722,6 +1773,7 @@ pub fn validate_embedding_settings(
         api_key,
         dimensions,
         document_template,
+        document_template_max_bytes,
         url,
         request,
         response,

milli/src/vector/mod.rs

@@ -305,6 +305,16 @@ impl Embedder {
             Embedder::Rest(embedder) => embedder.distribution(),
         }
     }
+
+    pub fn uses_document_template(&self) -> bool {
+        match self {
+            Embedder::HuggingFace(_)
+            | Embedder::OpenAi(_)
+            | Embedder::Ollama(_)
+            | Embedder::Rest(_) => true,
+            Embedder::UserProvided(_) => false,
+        }
+    }
 }
 
 /// Describes the mean and sigma of distribution of embedding similarity in the embedding space.

milli/src/vector/settings.rs

@@ -1,11 +1,12 @@
 use std::collections::BTreeMap;
+use std::num::NonZeroUsize;
 
 use deserr::Deserr;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
 
 use super::{ollama, openai, DistributionShift};
-use crate::prompt::PromptData;
+use crate::prompt::{default_max_bytes, PromptData};
 use crate::update::Setting;
 use crate::vector::EmbeddingConfig;
 use crate::UserError;
@@ -34,6 +35,9 @@ pub struct EmbeddingSettings {
     pub document_template: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    pub document_template_max_bytes: Setting<usize>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default)]
     pub url: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
@@ -111,6 +115,7 @@ impl SettingsDiff {
                     mut response,
                     mut distribution,
                     mut headers,
+                    mut document_template_max_bytes,
                 } = old;
 
                 let EmbeddingSettings {
@@ -125,6 +130,7 @@ impl SettingsDiff {
                     response: new_response,
                     distribution: new_distribution,
                     headers: new_headers,
+                    document_template_max_bytes: new_document_template_max_bytes,
                 } = new;
 
                 let mut reindex_action = None;
@@ -142,6 +148,7 @@ impl SettingsDiff {
                         &mut request,
                         &mut response,
                         &mut document_template,
+                        &mut document_template_max_bytes,
                         &mut headers,
                     )
                 }
@@ -190,6 +197,23 @@ impl SettingsDiff {
                     );
                 }
 
+                if document_template_max_bytes.apply(new_document_template_max_bytes) {
+                    let previous_document_template_max_bytes =
+                        document_template_max_bytes.set().unwrap_or(default_max_bytes().get());
+                    let new_document_template_max_bytes =
+                        new_document_template_max_bytes.set().unwrap_or(default_max_bytes().get());
+
+                    // only reindex if the size increased. Reasoning:
+                    // - size decrease is a performance optimization, so we don't reindex and we keep the more accurate vectors
+                    // - size increase is an accuracy optimization, so we want to reindex
+                    if new_document_template_max_bytes > previous_document_template_max_bytes {
+                        ReindexAction::push_action(
+                            &mut reindex_action,
+                            ReindexAction::RegeneratePrompts,
+                        )
+                    }
+                }
+
                 distribution.apply(new_distribution);
                 api_key.apply(new_api_key);
                 headers.apply(new_headers);
@@ -206,6 +230,7 @@ impl SettingsDiff {
                     response,
                     distribution,
                     headers,
+                    document_template_max_bytes,
                 };
 
                 match reindex_action {
@@ -239,6 +264,7 @@ fn apply_default_for_source(
     request: &mut Setting<serde_json::Value>,
     response: &mut Setting<serde_json::Value>,
     document_template: &mut Setting<String>,
+    document_template_max_bytes: &mut Setting<usize>,
     headers: &mut Setting<BTreeMap<String, String>>,
 ) {
     match source {
@@ -286,6 +312,7 @@ fn apply_default_for_source(
             *request = Setting::NotSet;
             *response = Setting::NotSet;
             *document_template = Setting::NotSet;
+            *document_template_max_bytes = Setting::NotSet;
             *headers = Setting::NotSet;
         }
         Setting::NotSet => {}
@@ -316,6 +343,7 @@ impl EmbeddingSettings {
     pub const API_KEY: &'static str = "apiKey";
     pub const DIMENSIONS: &'static str = "dimensions";
     pub const DOCUMENT_TEMPLATE: &'static str = "documentTemplate";
+    pub const DOCUMENT_TEMPLATE_MAX_BYTES: &'static str = "documentTemplateMaxBytes";
 
     pub const URL: &'static str = "url";
     pub const REQUEST: &'static str = "request";
@@ -459,6 +487,8 @@ impl std::fmt::Display for EmbedderSource {
 impl From<EmbeddingConfig> for EmbeddingSettings {
     fn from(value: EmbeddingConfig) -> Self {
         let EmbeddingConfig { embedder_options, prompt } = value;
+        let document_template_max_bytes =
+            Setting::Set(prompt.max_bytes.unwrap_or(default_max_bytes()).get());
         match embedder_options {
             super::EmbedderOptions::HuggingFace(super::hf::EmbedderOptions {
                 model,
@@ -471,6 +501,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::NotSet,
                 dimensions: Setting::NotSet,
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::NotSet,
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -490,6 +521,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::some_or_not_set(url),
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -509,6 +541,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::some_or_not_set(url),
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -525,6 +558,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::NotSet,
                 dimensions: Setting::Set(dimensions),
                 document_template: Setting::NotSet,
+                document_template_max_bytes: Setting::NotSet,
                 url: Setting::NotSet,
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -546,6 +580,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::Set(url),
                 request: Setting::Set(request),
                 response: Setting::Set(response),
@@ -566,6 +601,7 @@ impl From<EmbeddingSettings> for EmbeddingConfig {
             api_key,
             dimensions,
             document_template,
+            document_template_max_bytes,
             url,
             request,
             response,
@@ -648,7 +684,12 @@ impl From<EmbeddingSettings> for EmbeddingConfig {
         }
 
         if let Setting::Set(template) = document_template {
-            this.prompt = PromptData { template }
+            let max_bytes = document_template_max_bytes
+                .set()
+                .and_then(NonZeroUsize::new)
+                .unwrap_or(default_max_bytes());
+
+            this.prompt = PromptData { template, max_bytes: Some(max_bytes) }
         }
 
         this