Kind word

4939: Introduce the `STARTS WITH` filter operator r=irevoire a=Kerollmops

This PR fixes #4872 by introducing the `STARTS WITH` filter operator and gating it under the _contains filter_ experimental feature along with the `CONTAINS` one. I also updated [the experimental feature discussion page](https://github.com/orgs/meilisearch/discussions/763).

Co-authored-by: Clément Renault <clement@meilisearch.com>

Cargo.lock

@@ -471,7 +471,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "benchmarks"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "bytes",
@@ -527,7 +527,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash",
+ "rustc-hash 1.1.0",
  "shlex",
  "syn 2.0.60",
 ]
@@ -652,7 +652,7 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "time",
@@ -1622,7 +1622,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1834,7 +1834,7 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
 
 [[package]]
 name = "file-store"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "tempfile",
  "thiserror",
@@ -1856,7 +1856,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "insta",
  "nom",
@@ -1876,7 +1876,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2000,7 +2000,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "arbitrary",
  "clap",
@@ -2552,7 +2552,7 @@ checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
 
 [[package]]
 name = "index-scheduler"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "arroy",
@@ -2746,7 +2746,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2838,7 +2838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.4",
+ "windows-targets 0.48.1",
 ]
 
 [[package]]
@@ -3365,7 +3365,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "insta",
  "md5",
@@ -3374,7 +3374,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -3463,7 +3463,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "base64 0.22.1",
  "enum-iterator",
@@ -3482,7 +3482,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -3512,7 +3512,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -3520,6 +3520,7 @@ dependencies = [
  "file-store",
  "meilisearch-auth",
  "meilisearch-types",
+ "serde",
  "time",
  "uuid",
 ]
@@ -3542,7 +3543,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "arroy",
  "big_s",
@@ -3976,7 +3977,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "big_s",
  "serde_json",
@@ -4307,7 +4308,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash",
+ "rustc-hash 1.1.0",
  "rustls",
  "thiserror",
  "tokio",
@@ -4316,14 +4317,14 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.3"
+version = "0.11.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe"
+checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
 dependencies = [
  "bytes",
  "rand",
  "ring",
- "rustc-hash",
+ "rustc-hash 2.0.0",
  "rustls",
  "slab",
  "thiserror",
@@ -4696,6 +4697,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc-hash"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
@@ -4834,9 +4841,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
@@ -4852,9 +4859,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5347,7 +5354,7 @@ dependencies = [
  "fancy-regex 0.12.0",
  "lazy_static",
  "parking_lot",
- "rustc-hash",
+ "rustc-hash 1.1.0",
 ]
 
 [[package]]
@@ -6361,7 +6368,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.10.0"
+version = "1.11.0"
 dependencies = [
  "anyhow",
  "build-info",

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.10.0"
+version = "1.11.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

filter-parser/src/condition.rs

@@ -27,6 +27,7 @@ pub enum Condition<'a> {
     LowerThanOrEqual(Token<'a>),
     Between { from: Token<'a>, to: Token<'a> },
     Contains { keyword: Token<'a>, word: Token<'a> },
+    StartsWith { keyword: Token<'a>, word: Token<'a> },
 }
 
 /// condition      = value ("==" | ">" ...) value
@@ -121,6 +122,34 @@ pub fn parse_not_contains(input: Span) -> IResult<FilterCondition> {
     ))
 }
 
+/// starts with        = value "CONTAINS" value
+pub fn parse_starts_with(input: Span) -> IResult<FilterCondition> {
+    let (input, (fid, starts_with, value)) =
+        tuple((parse_value, tag("STARTS WITH"), cut(parse_value)))(input)?;
+    Ok((
+        input,
+        FilterCondition::Condition {
+            fid,
+            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
+        },
+    ))
+}
+
+/// starts with        = value "NOT" WS+ "CONTAINS" value
+pub fn parse_not_starts_with(input: Span) -> IResult<FilterCondition> {
+    let keyword = tuple((tag("NOT"), multispace1, tag("STARTS WITH")));
+    let (input, (fid, (_not, _spaces, starts_with), value)) =
+        tuple((parse_value, keyword, cut(parse_value)))(input)?;
+
+    Ok((
+        input,
+        FilterCondition::Not(Box::new(FilterCondition::Condition {
+            fid,
+            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
+        })),
+    ))
+}
+
 /// to             = value value "TO" WS+ value
 pub fn parse_to(input: Span) -> IResult<FilterCondition> {
     let (input, (key, from, _, _, to)) =

filter-parser/src/error.rs

@@ -146,7 +146,7 @@ impl<'a> Display for Error<'a> {
             }
             ErrorKind::InvalidPrimary => {
                 let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
-                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
             }
             ErrorKind::InvalidEscapedNumber => {
                 writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?

filter-parser/src/lib.rs

@@ -49,7 +49,7 @@ use std::fmt::Debug;
 pub use condition::{parse_condition, parse_to, Condition};
 use condition::{
     parse_contains, parse_exists, parse_is_empty, parse_is_not_empty, parse_is_not_null,
-    parse_is_null, parse_not_contains, parse_not_exists,
+    parse_is_null, parse_not_contains, parse_not_exists, parse_not_starts_with, parse_starts_with,
 };
 use error::{cut_with_err, ExpectedValueKind, NomErrorExt};
 pub use error::{Error, ErrorKind};
@@ -166,7 +166,8 @@ impl<'a> FilterCondition<'a> {
                 | Condition::LowerThan(_)
                 | Condition::LowerThanOrEqual(_)
                 | Condition::Between { .. } => None,
-                Condition::Contains { keyword, word: _ } => Some(keyword),
+                Condition::Contains { keyword, word: _ }
+                | Condition::StartsWith { keyword, word: _ } => Some(keyword),
             },
             FilterCondition::Not(this) => this.use_contains_operator(),
             FilterCondition::Or(seq) | FilterCondition::And(seq) => {
@@ -484,6 +485,8 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
         parse_to,
         parse_contains,
         parse_not_contains,
+        parse_starts_with,
+        parse_not_starts_with,
         // the next lines are only for error handling and are written at the end to have the less possible performance impact
         parse_geo,
         parse_geo_distance,
@@ -567,6 +570,7 @@ impl<'a> std::fmt::Display for Condition<'a> {
             Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"),
             Condition::Between { from, to } => write!(f, "{from} TO {to}"),
             Condition::Contains { word, keyword: _ } => write!(f, "CONTAINS {word}"),
+            Condition::StartsWith { word, keyword: _ } => write!(f, "STARTS WITH {word}"),
         }
     }
 }
@@ -680,6 +684,13 @@ pub mod tests {
         insta::assert_snapshot!(p("NOT subscribers NOT CONTAINS 'hello'"), @"{subscribers} CONTAINS {hello}");
         insta::assert_snapshot!(p("subscribers NOT   CONTAINS 'hello'"), @"NOT ({subscribers} CONTAINS {hello})");
 
+        // Test STARTS WITH + NOT STARTS WITH
+        insta::assert_snapshot!(p("subscribers STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
+        insta::assert_snapshot!(p("NOT subscribers STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
+        insta::assert_snapshot!(p("subscribers NOT STARTS WITH hel"), @"NOT ({subscribers} STARTS WITH {hel})");
+        insta::assert_snapshot!(p("NOT subscribers NOT STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
+        insta::assert_snapshot!(p("subscribers NOT   STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
+
         // Test nested NOT
         insta::assert_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
         insta::assert_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
@@ -751,7 +762,7 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("'OR'"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
         1:5 'OR'
         "###);
 
@@ -761,12 +772,12 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("channel Ponce"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
         1:14 channel Ponce
         "###);
 
         insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
         19:19 channel = Ponce OR
         "###);
 
@@ -851,12 +862,12 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
         "###);
 
         insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
         "###);
 
@@ -919,35 +930,35 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p(r#"value NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
         1:11 value NULL
         "###);
         insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
         1:15 value NOT NULL
         "###);
         insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
         1:12 value EMPTY
         "###);
         insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
         1:16 value NOT EMPTY
         "###);
         insta::assert_snapshot!(p(r#"value IS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
         1:9 value IS
         "###);
         insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
         1:13 value IS NOT
         "###);
         insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
         1:16 value IS EXISTS
         "###);
         insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
         1:20 value IS NOT EXISTS
         "###);
     }

filter-parser/src/value.rs

@@ -212,6 +212,8 @@ fn is_keyword(s: &str) -> bool {
             | "NULL"
             | "EMPTY"
             | "CONTAINS"
+            | "STARTS"
+            | "WITH"
             | "_geoRadius"
             | "_geoBoundingBox"
     )

index-scheduler/src/autobatcher.rs

@@ -25,8 +25,9 @@ enum AutobatchKind {
         primary_key: Option<String>,
     },
     DocumentEdition,
-    DocumentDeletion,
-    DocumentDeletionByFilter,
+    DocumentDeletion {
+        by_filter: bool,
+    },
     DocumentClear,
     Settings {
         allow_index_creation: bool,
@@ -65,10 +66,12 @@ impl From<KindWithContent> for AutobatchKind {
                 ..
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
             KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
-            KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
+            KindWithContent::DocumentDeletion { .. } => {
+                AutobatchKind::DocumentDeletion { by_filter: false }
+            }
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
             KindWithContent::DocumentDeletionByFilter { .. } => {
-                AutobatchKind::DocumentDeletionByFilter
+                AutobatchKind::DocumentDeletion { by_filter: true }
             }
             KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
                 AutobatchKind::Settings {
@@ -105,9 +108,7 @@ pub enum BatchKind {
     },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
-    },
-    DocumentDeletionByFilter {
-        id: TaskId,
+        includes_by_filter: bool,
     },
     ClearAndSettings {
         other: Vec<TaskId>,
@@ -205,12 +206,13 @@ impl BatchKind {
                 allow_index_creation,
             ),
             K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
-            K::DocumentDeletion => {
-                (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
-            }
-            K::DocumentDeletionByFilter => {
-                (Break(BatchKind::DocumentDeletionByFilter { id: task_id }), false)
-            }
+            K::DocumentDeletion { by_filter: includes_by_filter } => (
+                Continue(BatchKind::DocumentDeletion {
+                    deletion_ids: vec![task_id],
+                    includes_by_filter,
+                }),
+                false,
+            ),
             K::Settings { allow_index_creation } => (
                 Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
                 allow_index_creation,
@@ -228,7 +230,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition | K::DocumentDeletionByFilter) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -264,7 +266,7 @@ impl BatchKind {
             // The index deletion can batch with everything but must stop after
             (
                 BatchKind::DocumentClear { mut ids }
-                | BatchKind::DocumentDeletion { deletion_ids: mut ids }
+                | BatchKind::DocumentDeletion { deletion_ids: mut ids, includes_by_filter: _ }
                 | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
                 | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
                 K::IndexDeletion,
@@ -284,7 +286,7 @@ impl BatchKind {
 
             (
                 BatchKind::DocumentClear { mut ids },
-                K::DocumentClear | K::DocumentDeletion,
+                K::DocumentClear | K::DocumentDeletion { by_filter: _ },
             ) => {
                 ids.push(id);
                 Continue(BatchKind::DocumentClear { ids })
@@ -328,7 +330,7 @@ impl BatchKind {
             }
             (
                 BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
-                K::DocumentDeletion,
+                K::DocumentDeletion { by_filter: false },
             ) => {
                 operation_ids.push(id);
 
@@ -339,6 +341,13 @@ impl BatchKind {
                     operation_ids,
                 })
             }
+            // We can't batch a document operation with a delete by filter
+            (
+                this @ BatchKind::DocumentOperation { .. },
+                K::DocumentDeletion { by_filter: true },
+            ) => {
+                Break(this)
+            }
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
@@ -357,13 +366,18 @@ impl BatchKind {
                 operation_ids,
             }),
 
-            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentClear) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: _ }, K::DocumentClear) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
+            // we can't autobatch the deletion and import if the document deletion contained a filter
+            (
+                this @ BatchKind::DocumentDeletion { deletion_ids: _, includes_by_filter: true },
+                K::DocumentImport { .. }
+            ) => Break(this),
             // we can autobatch the deletion and import if the index already exists
             (
-                BatchKind::DocumentDeletion { mut deletion_ids },
+                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if index_already_exists => {
                 deletion_ids.push(id);
@@ -377,7 +391,7 @@ impl BatchKind {
             }
             // we can autobatch the deletion and import if both can't create an index
             (
-                BatchKind::DocumentDeletion { mut deletion_ids },
+                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if !allow_index_creation => {
                 deletion_ids.push(id);
@@ -396,9 +410,9 @@ impl BatchKind {
             ) => {
                 Break(this)
             }
-            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentDeletion) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter }, K::DocumentDeletion { by_filter }) => {
                 deletion_ids.push(id);
-                Continue(BatchKind::DocumentDeletion { deletion_ids })
+                Continue(BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: includes_by_filter | by_filter })
             }
             (this @ BatchKind::DocumentDeletion { .. }, K::Settings { .. }) => Break(this),
 
@@ -412,7 +426,7 @@ impl BatchKind {
             }),
             (
                 this @ BatchKind::Settings { .. },
-                K::DocumentImport { .. } | K::DocumentDeletion,
+                K::DocumentImport { .. } | K::DocumentDeletion { .. },
             ) => Break(this),
             (
                 BatchKind::Settings { mut settings_ids, allow_index_creation },
@@ -443,7 +457,7 @@ impl BatchKind {
                     settings_ids,
                     allow_index_creation,
                 },
-                K::DocumentDeletion,
+                K::DocumentDeletion { .. },
             ) => {
                 other.push(id);
                 Continue(BatchKind::ClearAndSettings {
@@ -505,7 +519,7 @@ impl BatchKind {
             // this MUST be AFTER the two previous branch
             (
                 this @ BatchKind::SettingsAndDocumentOperation { .. },
-                K::DocumentDeletion | K::DocumentImport { .. },
+                K::DocumentDeletion { .. } | K::DocumentImport { .. },
             ) => Break(this),
             (
                 BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
@@ -525,8 +539,7 @@ impl BatchKind {
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
                 | BatchKind::IndexSwap { .. }
-                | BatchKind::DocumentEdition { .. }
-                | BatchKind::DocumentDeletionByFilter { .. },
+                | BatchKind::DocumentEdition { .. },
                 _,
             ) => {
                 unreachable!()
@@ -616,6 +629,13 @@ mod tests {
         }
     }
 
+    fn doc_del_fil() -> KindWithContent {
+        KindWithContent::DocumentDeletionByFilter {
+            index_uid: String::from("doggo"),
+            filter_expr: serde_json::json!("cuteness > 100"),
+        }
+    }
+
     fn doc_clr() -> KindWithContent {
         KindWithContent::DocumentClear { index_uid: String::from("doggo") }
     }
@@ -676,10 +696,16 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple DocumentDeletion together
-        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
+
+        // we can autobatch one or multiple DocumentDeletionByFilter together
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
 
         // we can autobatch one or multiple Settings together
         debug_snapshot!(autobatch_from(true, None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
@@ -722,25 +748,63 @@ mod tests {
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
+
+        // But we can't autobatch document addition with document deletion by filter
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
+        // And the other way around
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
     fn simple_document_operation_dont_autobatch_with_other() {
-        // addition, updates and deletion can't batch together
+        // addition, updates and deletion by filter can't batch together
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
@@ -807,6 +871,7 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -816,6 +881,7 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -827,6 +893,7 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(true, None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -836,6 +903,7 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -901,10 +969,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
 
         // batch deletion and addition
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
     }
 
     #[test]

index-scheduler/src/batch.rs

@@ -110,9 +110,9 @@ pub(crate) enum IndexOperation {
         index_uid: String,
         task: Task,
     },
-    IndexDocumentDeletionByFilter {
+    DocumentDeletion {
         index_uid: String,
-        task: Task,
+        tasks: Vec<Task>,
     },
     DocumentClear {
         index_uid: String,
@@ -165,11 +165,11 @@ impl Batch {
             Batch::IndexOperation { op, .. } => match op {
                 IndexOperation::DocumentOperation { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
+                | IndexOperation::DocumentDeletion { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
                 }
-                IndexOperation::DocumentEdition { task, .. }
-                | IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
+                IndexOperation::DocumentEdition { task, .. } => {
                     RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
                 }
                 IndexOperation::SettingsAndDocumentOperation {
@@ -234,7 +234,7 @@ impl IndexOperation {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
             | IndexOperation::DocumentEdition { index_uid, .. }
-            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
+            | IndexOperation::DocumentDeletion { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -252,8 +252,8 @@ impl fmt::Display for IndexOperation {
             IndexOperation::DocumentEdition { .. } => {
                 f.write_str("IndexOperation::DocumentEdition")
             }
-            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
-                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
+            IndexOperation::DocumentDeletion { .. } => {
+                f.write_str("IndexOperation::DocumentDeletion")
             }
             IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
             IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
@@ -289,21 +289,6 @@ impl IndexScheduler {
                 },
                 must_create_index,
             })),
-            BatchKind::DocumentDeletionByFilter { id } => {
-                let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
-                match &task.kind {
-                    KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
-                        Ok(Some(Batch::IndexOperation {
-                            op: IndexOperation::IndexDocumentDeletionByFilter {
-                                index_uid: index_uid.clone(),
-                                task,
-                            },
-                            must_create_index: false,
-                        }))
-                    }
-                    _ => unreachable!(),
-                }
-            }
             BatchKind::DocumentEdition { id } => {
                 let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 match &task.kind {
@@ -366,30 +351,11 @@ impl IndexScheduler {
                     must_create_index,
                 }))
             }
-            BatchKind::DocumentDeletion { deletion_ids } => {
+            BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: _ } => {
                 let tasks = self.get_existing_tasks(rtxn, deletion_ids)?;
 
-                let mut operations = Vec::with_capacity(tasks.len());
-                let mut documents_counts = Vec::with_capacity(tasks.len());
-                for task in &tasks {
-                    match task.kind {
-                        KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
-                            operations.push(DocumentOperation::Delete(documents_ids.clone()));
-                            documents_counts.push(documents_ids.len() as u64);
-                        }
-                        _ => unreachable!(),
-                    }
-                }
-
                 Ok(Some(Batch::IndexOperation {
-                    op: IndexOperation::DocumentOperation {
-                        index_uid,
-                        primary_key: None,
-                        method: IndexDocumentsMethod::ReplaceDocuments,
-                        documents_counts,
-                        operations,
-                        tasks,
-                    },
+                    op: IndexOperation::DocumentDeletion { index_uid, tasks },
                     must_create_index,
                 }))
             }
@@ -1439,7 +1405,7 @@ impl IndexScheduler {
                 {
                     (original_filter, context, function)
                 } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
+                    // In the case of a `documentEdition` the details MUST be set
                     unreachable!();
                 };
 
@@ -1469,52 +1435,102 @@ impl IndexScheduler {
 
                 Ok(vec![task])
             }
-            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
-                let filter =
-                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
-                        &task.kind
-                    {
-                        filter_expr
-                    } else {
-                        unreachable!()
-                    };
-                let deleted_documents = delete_document_by_filter(
-                    index_wtxn,
-                    filter,
-                    self.index_mapper.indexer_config(),
-                    self.must_stop_processing.clone(),
-                    index,
-                );
-                let original_filter = if let Some(Details::DocumentDeletionByFilter {
-                    original_filter,
-                    deleted_documents: _,
-                }) = task.details
-                {
-                    original_filter
-                } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
-                    unreachable!();
-                };
+            IndexOperation::DocumentDeletion { mut tasks, index_uid: _ } => {
+                let mut to_delete = RoaringBitmap::new();
+                let external_documents_ids = index.external_documents_ids();
 
-                match deleted_documents {
-                    Ok(deleted_documents) => {
-                        task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(deleted_documents),
-                        });
-                    }
-                    Err(e) => {
-                        task.status = Status::Failed;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(0),
-                        });
-                        task.error = Some(e.into());
+                for task in tasks.iter_mut() {
+                    let before = to_delete.len();
+                    task.status = Status::Succeeded;
+
+                    match &task.kind {
+                        KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
+                            for id in documents_ids {
+                                if let Some(id) = external_documents_ids.get(index_wtxn, id)? {
+                                    to_delete.insert(id);
+                                }
+                            }
+                            let will_be_removed = to_delete.len() - before;
+                            task.details = Some(Details::DocumentDeletion {
+                                provided_ids: documents_ids.len(),
+                                deleted_documents: Some(will_be_removed),
+                            });
+                        }
+                        KindWithContent::DocumentDeletionByFilter { index_uid: _, filter_expr } => {
+                            let before = to_delete.len();
+                            let filter = match Filter::from_json(filter_expr) {
+                                Ok(filter) => filter,
+                                Err(err) => {
+                                    // theorically, this should be catched by deserr before reaching the index-scheduler and cannot happens
+                                    task.status = Status::Failed;
+                                    task.error = match err {
+                                        milli::Error::UserError(
+                                            milli::UserError::InvalidFilterExpression { .. },
+                                        ) => Some(
+                                            Error::from(err)
+                                                .with_custom_error_code(Code::InvalidDocumentFilter)
+                                                .into(),
+                                        ),
+                                        e => Some(e.into()),
+                                    };
+                                    None
+                                }
+                            };
+                            if let Some(filter) = filter {
+                                let candidates =
+                                    filter.evaluate(index_wtxn, index).map_err(|err| match err {
+                                        milli::Error::UserError(
+                                            milli::UserError::InvalidFilter(_),
+                                        ) => Error::from(err)
+                                            .with_custom_error_code(Code::InvalidDocumentFilter),
+                                        e => e.into(),
+                                    });
+                                match candidates {
+                                    Ok(candidates) => to_delete |= candidates,
+                                    Err(err) => {
+                                        task.status = Status::Failed;
+                                        task.error = Some(err.into());
+                                    }
+                                };
+                            }
+                            let will_be_removed = to_delete.len() - before;
+                            if let Some(Details::DocumentDeletionByFilter {
+                                original_filter: _,
+                                deleted_documents,
+                            }) = &mut task.details
+                            {
+                                *deleted_documents = Some(will_be_removed);
+                            } else {
+                                // In the case of a `documentDeleteByFilter` the details MUST be set
+                                unreachable!()
+                            }
+                        }
+                        _ => unreachable!(),
                     }
                 }
 
-                Ok(vec![task])
+                let config = IndexDocumentsConfig {
+                    update_method: IndexDocumentsMethod::ReplaceDocuments,
+                    ..Default::default()
+                };
+
+                let must_stop_processing = self.must_stop_processing.clone();
+                let mut builder = milli::update::IndexDocuments::new(
+                    index_wtxn,
+                    index,
+                    self.index_mapper.indexer_config(),
+                    config,
+                    |indexing_step| tracing::debug!(update = ?indexing_step),
+                    || must_stop_processing.get(),
+                )?;
+
+                let (new_builder, _count) =
+                    builder.remove_documents_from_db_no_batch(&to_delete)?;
+                builder = new_builder;
+
+                let _ = builder.execute()?;
+
+                Ok(tasks)
             }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
@@ -1718,46 +1734,6 @@ impl IndexScheduler {
     }
 }
 
-fn delete_document_by_filter<'a>(
-    wtxn: &mut RwTxn<'a>,
-    filter: &serde_json::Value,
-    indexer_config: &IndexerConfig,
-    must_stop_processing: MustStopProcessing,
-    index: &'a Index,
-) -> Result<u64> {
-    let filter = Filter::from_json(filter)?;
-    Ok(if let Some(filter) = filter {
-        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
-            milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
-                Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
-            }
-            e => e.into(),
-        })?;
-
-        let config = IndexDocumentsConfig {
-            update_method: IndexDocumentsMethod::ReplaceDocuments,
-            ..Default::default()
-        };
-
-        let mut builder = milli::update::IndexDocuments::new(
-            wtxn,
-            index,
-            indexer_config,
-            config,
-            |indexing_step| tracing::debug!(update = ?indexing_step),
-            || must_stop_processing.get(),
-        )?;
-
-        let (new_builder, count) = builder.remove_documents_from_db_no_batch(&candidates)?;
-        builder = new_builder;
-
-        let _ = builder.execute()?;
-        count
-    } else {
-        0
-    })
-}
-
 fn edit_documents_by_function<'a>(
     wtxn: &mut RwTxn<'a>,
     filter: &Option<serde_json::Value>,

index-scheduler/src/features.rs

@@ -87,7 +87,7 @@ impl RoFeatures {
             Ok(())
         } else {
             Err(FeatureNotEnabledError {
-                disabled_action: "Using `CONTAINS` in a filter",
+                disabled_action: "Using `CONTAINS` or `STARTS WITH` in a filter",
                 feature: "contains filter",
                 issue_link: "https://github.com/orgs/meilisearch/discussions/763",
             }

index-scheduler/src/insta_snapshot.rs

@@ -11,6 +11,9 @@ use crate::index_mapper::IndexMapper;
 use crate::{IndexScheduler, Kind, Status, BEI128};
 
 pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
+    // Since we'll snapshot the index right afterward, we don't need to ensure it's internally consistent for every run.
+    // We can only do it for the release run, where the function runs way faster.
+    #[cfg(not(debug_assertions))]
     scheduler.assert_internally_consistent();
 
     let IndexScheduler {

index-scheduler/src/lib.rs

@@ -35,6 +35,7 @@ pub type TaskId = u32;
 use std::collections::{BTreeMap, HashMap};
 use std::io::{self, BufReader, Read};
 use std::ops::{Bound, RangeBounds};
+use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::Ordering::{self, Relaxed};
 use std::sync::atomic::{AtomicBool, AtomicU32};
@@ -612,19 +613,24 @@ impl IndexScheduler {
                 #[cfg(test)]
                 run.breakpoint(Breakpoint::Init);
 
-                run.wake_up.wait();
+                run.wake_up.wait_timeout(std::time::Duration::from_secs(60));
 
                 loop {
-                    match run.tick() {
-                        Ok(TickOutcome::TickAgain(_)) => (),
-                        Ok(TickOutcome::WaitForSignal) => run.wake_up.wait(),
-                        Err(e) => {
+                    let ret = catch_unwind(AssertUnwindSafe(|| run.tick()));
+                    match ret {
+                        Ok(Ok(TickOutcome::TickAgain(_))) => (),
+                        Ok(Ok(TickOutcome::WaitForSignal)) => run.wake_up.wait(),
+                        Ok(Err(e)) => {
                             tracing::error!("{e}");
                             // Wait one second when an irrecoverable error occurs.
                             if !e.is_recoverable() {
                                 std::thread::sleep(Duration::from_secs(1));
                             }
                         }
+                        Err(_panic) => {
+                            tracing::error!("Internal error: Unexpected panic in the `IndexScheduler::run` method.");
+
+                        }
                     }
                 }
             })
@@ -1758,6 +1764,7 @@ mod tests {
     use crossbeam::channel::RecvTimeoutError;
     use file_store::File;
     use insta::assert_json_snapshot;
+    use maplit::btreeset;
     use meili_snap::{json_string, snapshot};
     use meilisearch_auth::AuthFilter;
     use meilisearch_types::document_formats::DocumentFormatError;
@@ -2000,11 +2007,13 @@ mod tests {
         fn advance_till(&mut self, breakpoints: impl IntoIterator<Item = Breakpoint>) {
             for breakpoint in breakpoints {
                 let b = self.advance();
-                let state = snapshot_index_scheduler(&self.index_scheduler);
                 assert_eq!(
-                    b, breakpoint,
-                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{state}",
-                    breakpoint, b
+                    b,
+                    breakpoint,
+                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{}",
+                    breakpoint,
+                    b,
+                    snapshot_index_scheduler(&self.index_scheduler)
                 );
             }
         }
@@ -2028,7 +2037,6 @@ mod tests {
         // Wait for one successful batch.
         #[track_caller]
         fn advance_one_successful_batch(&mut self) {
-            self.index_scheduler.assert_internally_consistent();
             self.advance_till([Start, BatchCreated]);
             loop {
                 match self.advance() {
@@ -2047,7 +2055,6 @@ mod tests {
             }
 
             self.advance_till([AfterProcessing]);
-            self.index_scheduler.assert_internally_consistent();
         }
 
         // Wait for one failed batch.
@@ -2547,6 +2554,117 @@ mod tests {
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
 
+    #[test]
+    fn fail_in_process_batch_for_document_deletion() {
+        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+        use meilisearch_types::settings::{Settings, Unchecked};
+        let mut new_settings: Box<Settings<Unchecked>> = Box::default();
+        new_settings.filterable_attributes = Setting::Set(btreeset!(S("catto")));
+
+        index_scheduler
+            .register(
+                KindWithContent::SettingsUpdate {
+                    index_uid: S("doggos"),
+                    new_settings,
+                    is_deletion: false,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+
+        let content = r#"[
+            { "id": 1, "doggo": "jean bob" },
+            { "id": 2, "catto": "jorts" },
+            { "id": 3, "doggo": "bork" }
+        ]"#;
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
+        let documents_count = read_json(content.as_bytes(), &mut file).unwrap();
+        file.persist().unwrap();
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: Some(S("id")),
+                    method: ReplaceDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_setting_and_document_addition");
+
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_settings");
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_documents");
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletion {
+                    index_uid: S("doggos"),
+                    documents_ids: vec![S("1")],
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        // This one should not be catched by Meilisearch but it's still nice to handle it because if one day we break the filters it could happens
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!(true),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        // Should fail because the ids are not filterable
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!("id = 2"),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletionByFilter {
+                    index_uid: S("doggos"),
+                    filter_expr: serde_json::json!("catto EXISTS"),
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_document_deletions");
+
+        // Everything should be batched together
+        handle.advance_one_successful_batch();
+        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_documents");
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents_remaining_should_only_be_bork");
+    }
+
     #[test]
     fn do_not_batch_task_of_different_indexes() {
         let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
@@ -5397,6 +5515,9 @@ mod tests {
                     ),
                     prompt: PromptData {
                         template: "{{doc.doggo}}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[1, 2]>,
@@ -5611,7 +5732,10 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[0]>,
@@ -5651,7 +5775,10 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                        max_bytes: Some(
+                            400,
+                        ),
                     },
                 },
                 user_provided: RoaringBitmap<[]>,

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap

@@ -0,0 +1,44 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap

@@ -0,0 +1,43 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [1,]
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap

@@ -0,0 +1,43 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [1,]
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap

@@ -0,0 +1,56 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+2 {uid: 2, status: succeeded, details: { received_document_ids: 1, deleted_documents: Some(1) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
+3 {uid: 3, status: failed, error: ResponseError { code: 200, message: "Invalid type for filter subexpression: expected: String, Array, found: true.", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: true, deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
+4 {uid: 4, status: failed, error: ResponseError { code: 200, message: "Attribute `id` is not filterable. Available filterable attributes are: `catto`.\n1:3 id = 2", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: "id = 2", deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
+5 {uid: 5, status: succeeded, details: { original_filter: "catto EXISTS", deleted_documents: Some(1) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,2,5,]
+failed [3,4,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"documentDeletion" [2,3,4,5,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,3,4,5,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+[timestamp] [4,]
+[timestamp] [5,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,3,4,5,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,3,4,5,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap

@@ -0,0 +1,9 @@
+---
+source: index-scheduler/src/lib.rs
+---
+[
+  {
+    "id": 3,
+    "doggo": "bork"
+  }
+]

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap

@@ -0,0 +1,53 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+2 {uid: 2, status: enqueued, details: { received_document_ids: 1, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
+3 {uid: 3, status: enqueued, details: { original_filter: true, deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
+4 {uid: 4, status: enqueued, details: { original_filter: "id = 2", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
+5 {uid: 5, status: enqueued, details: { original_filter: "catto EXISTS", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
+----------------------------------------------------------------------
+### Status:
+enqueued [2,3,4,5,]
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"documentDeletion" [2,3,4,5,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,3,4,5,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+[timestamp] [3,]
+[timestamp] [4,]
+[timestamp] [5,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap

@@ -0,0 +1,39 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

meilisearch-types/src/error.rs

@@ -238,8 +238,14 @@ InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacets              , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacetsByIndex       , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacetOrder          , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederated           , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchFederationOptions   , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchMaxValuesPerFacet   , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchMergeFacets         , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryFacets         , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchQueryPagination     , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchQueryRankingRules   , InvalidRequest       , BAD_REQUEST ;
 InvalidMultiSearchWeight              , InvalidRequest       , BAD_REQUEST ;
@@ -388,6 +394,7 @@ impl ErrorCode for milli::Error {
                     | UserError::InvalidOpenAiModelDimensionsMax { .. }
                     | UserError::InvalidSettingsDimensions { .. }
                     | UserError::InvalidUrl { .. }
+                    | UserError::InvalidSettingsDocumentTemplateMaxBytes { .. }
                     | UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders,
                     UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
                     UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,

meilisearch-types/src/index_uid.rs

@@ -1,3 +1,4 @@
+use std::borrow::Borrow;
 use std::error::Error;
 use std::fmt;
 use std::str::FromStr;
@@ -8,7 +9,7 @@ use crate::error::{Code, ErrorCode};
 
 /// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400
 /// bytes long
-#[derive(Debug, Clone, PartialEq, Eq, Deserr)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord)]
 #[deserr(try_from(String) = IndexUid::try_from -> IndexUidFormatError)]
 pub struct IndexUid(String);
 
@@ -70,6 +71,12 @@ impl From<IndexUid> for String {
     }
 }
 
+impl Borrow<String> for IndexUid {
+    fn borrow(&self) -> &String {
+        &self.0
+    }
+}
+
 #[derive(Debug)]
 pub struct IndexUidFormatError {
     pub invalid_uid: String,

meilisearch-types/src/locales.rs

@@ -1,134 +1,6 @@
 use deserr::Deserr;
 use milli::LocalizedAttributesRule;
 use serde::{Deserialize, Serialize};
-use serde_json::json;
-
-/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
-///
-/// this enum implements `Deserr` in order to be used in the API.
-macro_rules! make_locale {
-
-    ($($language:tt), +) => {
-        #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
-        #[deserr(rename_all = camelCase)]
-        #[serde(rename_all = "camelCase")]
-        pub enum Locale {
-            $($language),+,
-        }
-
-        impl From<milli::tokenizer::Language> for Locale {
-            fn from(other: milli::tokenizer::Language) -> Locale {
-                match other {
-                    $(milli::tokenizer::Language::$language => Locale::$language), +
-                }
-            }
-        }
-
-        impl From<Locale> for milli::tokenizer::Language {
-            fn from(other: Locale) -> milli::tokenizer::Language {
-                match other {
-                    $(Locale::$language => milli::tokenizer::Language::$language), +,
-                }
-            }
-        }
-
-        #[derive(Debug)]
-        pub struct LocaleFormatError {
-            pub invalid_locale: String,
-        }
-
-        impl std::fmt::Display for LocaleFormatError {
-            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                let valid_locales = [$(Locale::$language),+].iter().map(|l| format!("`{}`", json!(l).as_str().unwrap())).collect::<Vec<_>>().join(", ");
-                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
-            }
-        }
-    };
-}
-
-make_locale! {
-    Epo,
-    Eng,
-    Rus,
-    Cmn,
-    Spa,
-    Por,
-    Ita,
-    Ben,
-    Fra,
-    Deu,
-    Ukr,
-    Kat,
-    Ara,
-    Hin,
-    Jpn,
-    Heb,
-    Yid,
-    Pol,
-    Amh,
-    Jav,
-    Kor,
-    Nob,
-    Dan,
-    Swe,
-    Fin,
-    Tur,
-    Nld,
-    Hun,
-    Ces,
-    Ell,
-    Bul,
-    Bel,
-    Mar,
-    Kan,
-    Ron,
-    Slv,
-    Hrv,
-    Srp,
-    Mkd,
-    Lit,
-    Lav,
-    Est,
-    Tam,
-    Vie,
-    Urd,
-    Tha,
-    Guj,
-    Uzb,
-    Pan,
-    Aze,
-    Ind,
-    Tel,
-    Pes,
-    Mal,
-    Ori,
-    Mya,
-    Nep,
-    Sin,
-    Khm,
-    Tuk,
-    Aka,
-    Zul,
-    Sna,
-    Afr,
-    Lat,
-    Slk,
-    Cat,
-    Tgl,
-    Hye
-}
-
-impl std::error::Error for LocaleFormatError {}
-
-impl std::str::FromStr for Locale {
-    type Err = LocaleFormatError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        milli::tokenizer::Language::from_code(s)
-            .map(Self::from)
-            .ok_or(LocaleFormatError { invalid_locale: s.to_string() })
-    }
-}
 
 #[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)]
 #[deserr(rename_all = camelCase)]
@@ -155,3 +27,135 @@ impl From<LocalizedAttributesRuleView> for LocalizedAttributesRule {
         }
     }
 }
+
+/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
+///
+/// this enum implements `Deserr` in order to be used in the API.
+macro_rules! make_locale {
+    ($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => {
+        #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
+        #[deserr(rename_all = camelCase)]
+        #[serde(rename_all = "camelCase")]
+        pub enum Locale {
+            $($iso_639_1,)+
+            $($iso_639_3,)+
+        }
+
+        impl From<milli::tokenizer::Language> for Locale {
+            fn from(other: milli::tokenizer::Language) -> Locale {
+                match other {
+                    $(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
+                }
+            }
+        }
+
+        impl From<Locale> for milli::tokenizer::Language {
+            fn from(other: Locale) -> milli::tokenizer::Language {
+                match other {
+                    $(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
+                    $(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
+                }
+            }
+        }
+
+        impl std::str::FromStr for Locale {
+            type Err = LocaleFormatError;
+
+            fn from_str(s: &str) -> Result<Self, Self::Err> {
+                let locale = match s {
+                    $($iso_639_1_str => Locale::$iso_639_1,)+
+                    $($iso_639_3_str => Locale::$iso_639_3,)+
+                    _ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
+                };
+
+                Ok(locale)
+            }
+        }
+
+        #[derive(Debug)]
+        pub struct LocaleFormatError {
+            pub invalid_locale: String,
+        }
+
+        impl std::fmt::Display for LocaleFormatError {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                let valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+].join(", ");
+                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales)
+            }
+        }
+
+        impl std::error::Error for LocaleFormatError {}
+    };
+}
+
+make_locale!(
+    (Af, "af") => (Afr, "afr"),
+    (Ak, "ak") => (Aka, "aka"),
+    (Am, "am") => (Amh, "amh"),
+    (Ar, "ar") => (Ara, "ara"),
+    (Az, "az") => (Aze, "aze"),
+    (Be, "be") => (Bel, "bel"),
+    (Bn, "bn") => (Ben, "ben"),
+    (Bg, "bg") => (Bul, "bul"),
+    (Ca, "ca") => (Cat, "cat"),
+    (Cs, "cs") => (Ces, "ces"),
+    (Zh, "zh") => (Cmn, "cmn"),
+    (Da, "da") => (Dan, "dan"),
+    (De, "de") => (Deu, "deu"),
+    (El, "el") => (Ell, "ell"),
+    (En, "en") => (Eng, "eng"),
+    (Eo, "eo") => (Epo, "epo"),
+    (Et, "et") => (Est, "est"),
+    (Fi, "fi") => (Fin, "fin"),
+    (Fr, "fr") => (Fra, "fra"),
+    (Gu, "gu") => (Guj, "guj"),
+    (He, "he") => (Heb, "heb"),
+    (Hi, "hi") => (Hin, "hin"),
+    (Hr, "hr") => (Hrv, "hrv"),
+    (Hu, "hu") => (Hun, "hun"),
+    (Hy, "hy") => (Hye, "hye"),
+    (Id, "id") => (Ind, "ind"),
+    (It, "it") => (Ita, "ita"),
+    (Jv, "jv") => (Jav, "jav"),
+    (Ja, "ja") => (Jpn, "jpn"),
+    (Kn, "kn") => (Kan, "kan"),
+    (Ka, "ka") => (Kat, "kat"),
+    (Km, "km") => (Khm, "khm"),
+    (Ko, "ko") => (Kor, "kor"),
+    (La, "la") => (Lat, "lat"),
+    (Lv, "lv") => (Lav, "lav"),
+    (Lt, "lt") => (Lit, "lit"),
+    (Ml, "ml") => (Mal, "mal"),
+    (Mr, "mr") => (Mar, "mar"),
+    (Mk, "mk") => (Mkd, "mkd"),
+    (My, "my") => (Mya, "mya"),
+    (Ne, "ne") => (Nep, "nep"),
+    (Nl, "nl") => (Nld, "nld"),
+    (Nb, "nb") => (Nob, "nob"),
+    (Or, "or") => (Ori, "ori"),
+    (Pa, "pa") => (Pan, "pan"),
+    (Fa, "fa") => (Pes, "pes"),
+    (Pl, "pl") => (Pol, "pol"),
+    (Pt, "pt") => (Por, "por"),
+    (Ro, "ro") => (Ron, "ron"),
+    (Ru, "ru") => (Rus, "rus"),
+    (Si, "si") => (Sin, "sin"),
+    (Sk, "sk") => (Slk, "slk"),
+    (Sl, "sl") => (Slv, "slv"),
+    (Sn, "sn") => (Sna, "sna"),
+    (Es, "es") => (Spa, "spa"),
+    (Sr, "sr") => (Srp, "srp"),
+    (Sv, "sv") => (Swe, "swe"),
+    (Ta, "ta") => (Tam, "tam"),
+    (Te, "te") => (Tel, "tel"),
+    (Tl, "tl") => (Tgl, "tgl"),
+    (Th, "th") => (Tha, "tha"),
+    (Tk, "tk") => (Tuk, "tuk"),
+    (Tr, "tr") => (Tur, "tur"),
+    (Uk, "uk") => (Ukr, "ukr"),
+    (Ur, "ur") => (Urd, "urd"),
+    (Uz, "uz") => (Uzb, "uzb"),
+    (Vi, "vi") => (Vie, "vie"),
+    (Yi, "yi") => (Yid, "yid"),
+    (Zu, "zu") => (Zul, "zul"),
+);

meilisearch-types/src/versioning.rs

@@ -10,38 +10,52 @@ static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
 static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
 
 /// Persists the version of the current Meilisearch binary to a VERSION file
-pub fn create_version_file(db_path: &Path) -> io::Result<()> {
+pub fn create_current_version_file(db_path: &Path) -> io::Result<()> {
+    create_version_file(db_path, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
+}
+
+pub fn create_version_file(
+    db_path: &Path,
+    major: &str,
+    minor: &str,
+    patch: &str,
+) -> io::Result<()> {
     let version_path = db_path.join(VERSION_FILE_NAME);
-    fs::write(version_path, format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH))
+    fs::write(version_path, format!("{}.{}.{}", major, minor, patch))
 }
 
 /// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
 pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
-    let version_path = db_path.join(VERSION_FILE_NAME);
+    let (major, minor, patch) = get_version(db_path)?;
 
-    match fs::read_to_string(version_path) {
-        Ok(version) => {
-            let version_components = version.split('.').collect::<Vec<_>>();
-            let (major, minor, patch) = match &version_components[..] {
-                [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
-                _ => return Err(VersionFileError::MalformedVersionFile.into()),
-            };
-
-            if major != VERSION_MAJOR || minor != VERSION_MINOR {
-                return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
-            }
-        }
-        Err(error) => {
-            return match error.kind() {
-                ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()),
-                _ => Err(error.into()),
-            }
-        }
+    if major != VERSION_MAJOR || minor != VERSION_MINOR {
+        return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
     }
 
     Ok(())
 }
 
+pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> {
+    let version_path = db_path.join(VERSION_FILE_NAME);
+
+    match fs::read_to_string(version_path) {
+        Ok(version) => parse_version(&version),
+        Err(error) => match error.kind() {
+            ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile),
+            _ => Err(error.into()),
+        },
+    }
+}
+
+pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> {
+    let version_components = version.split('.').collect::<Vec<_>>();
+    let (major, minor, patch) = match &version_components[..] {
+        [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
+        _ => return Err(VersionFileError::MalformedVersionFile),
+    };
+    Ok((major, minor, patch))
+}
+
 #[derive(thiserror::Error, Debug)]
 pub enum VersionFileError {
     #[error(
@@ -58,4 +72,7 @@ pub enum VersionFileError {
         env!("CARGO_PKG_VERSION").to_string()
     )]
     VersionMismatch { major: String, minor: String, patch: String },
+
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
 }

meilisearch/src/error.rs

@@ -4,6 +4,7 @@ use byte_unit::{Byte, UnitType};
 use meilisearch_types::document_formats::{DocumentFormatError, PayloadType};
 use meilisearch_types::error::{Code, ErrorCode, ResponseError};
 use meilisearch_types::index_uid::{IndexUid, IndexUidFormatError};
+use meilisearch_types::milli::OrderBy;
 use serde_json::Value;
 use tokio::task::JoinError;
 
@@ -27,10 +28,20 @@ pub enum MeilisearchHttpError {
     EmptyFilter,
     #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
     InvalidExpression(&'static [&'static str], Value),
-    #[error("Using `federationOptions` is not allowed in a non-federated search.\n Hint: remove `federationOptions` from query #{0} or add `federation: {{}}` to the request.")]
+    #[error("Using `federationOptions` is not allowed in a non-federated search.\n - Hint: remove `federationOptions` from query #{0} or add `federation` to the request.")]
     FederationOptionsInNonFederatedRequest(usize),
-    #[error("Inside `.queries[{0}]`: Using pagination options is not allowed in federated queries.\n Hint: remove `{1}` from query #{0} or remove `federation: {{}}` from the request")]
+    #[error("Inside `.queries[{0}]`: Using pagination options is not allowed in federated queries.\n - Hint: remove `{1}` from query #{0} or remove `federation` from the request\n - Hint: pass `federation.limit` and `federation.offset` for pagination in federated search")]
     PaginationInFederatedQuery(usize, &'static str),
+    #[error("Inside `.queries[{0}]`: Using facet options is not allowed in federated queries.\n - Hint: remove `facets` from query #{0} or remove `federation` from the request\n - Hint: pass `federation.facetsByIndex.{1}: {2:?}` for facets in federated search")]
+    FacetsInFederatedQuery(usize, String, Vec<String>),
+    #[error("Inconsistent order for values in facet `{facet}`: index `{previous_uid}` orders {previous_facet_order}, but index `{current_uid}` orders {index_facet_order}.\n - Hint: Remove `federation.mergeFacets` or change `faceting.sortFacetValuesBy` to be consistent in settings.")]
+    InconsistentFacetOrder {
+        facet: String,
+        previous_facet_order: OrderBy,
+        previous_uid: String,
+        index_facet_order: OrderBy,
+        current_uid: String,
+    },
     #[error("A {0} payload is missing.")]
     MissingPayload(PayloadType),
     #[error("Too many search requests running at the same time: {0}. Retry after 10s.")]
@@ -96,6 +107,10 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::PaginationInFederatedQuery(_, _) => {
                 Code::InvalidMultiSearchQueryPagination
             }
+            MeilisearchHttpError::FacetsInFederatedQuery(..) => Code::InvalidMultiSearchQueryFacets,
+            MeilisearchHttpError::InconsistentFacetOrder { .. } => {
+                Code::InvalidMultiSearchFacetOrder
+            }
         }
     }
 }

meilisearch/src/lib.rs

@@ -13,11 +13,10 @@ pub mod search_queue;
 
 use std::fs::File;
 use std::io::{BufReader, BufWriter};
-use std::num::NonZeroUsize;
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::thread::{self, available_parallelism};
+use std::thread;
 use std::time::Duration;
 
 use actix_cors::Cors;
@@ -37,7 +36,7 @@ use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchR
 use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::KindWithContent;
-use meilisearch_types::versioning::{check_version_file, create_version_file};
+use meilisearch_types::versioning::{check_version_file, create_current_version_file};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
@@ -118,6 +117,7 @@ pub type LogStderrType = tracing_subscriber::filter::Filtered<
 pub fn create_app(
     index_scheduler: Data<IndexScheduler>,
     auth_controller: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
     opt: Opt,
     logs: (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
@@ -137,6 +137,7 @@ pub fn create_app(
                 s,
                 index_scheduler.clone(),
                 auth_controller.clone(),
+                search_queue.clone(),
                 &opt,
                 logs,
                 analytics.clone(),
@@ -318,7 +319,7 @@ fn open_or_create_database_unchecked(
     match (
         index_scheduler_builder(),
         auth_controller.map_err(anyhow::Error::from),
-        create_version_file(&opt.db_path).map_err(anyhow::Error::from),
+        create_current_version_file(&opt.db_path).map_err(anyhow::Error::from),
     ) {
         (Ok(i), Ok(a), Ok(())) => Ok((i, a)),
         (Err(e), _, _) | (_, Err(e), _) | (_, _, Err(e)) => {
@@ -469,19 +470,16 @@ pub fn configure_data(
     config: &mut web::ServiceConfig,
     index_scheduler: Data<IndexScheduler>,
     auth: Data<AuthController>,
+    search_queue: Data<SearchQueue>,
     opt: &Opt,
     (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
 ) {
-    let search_queue = SearchQueue::new(
-        opt.experimental_search_queue_size,
-        available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
-    );
     let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)
-        .app_data(web::Data::new(search_queue))
+        .app_data(search_queue)
         .app_data(web::Data::from(analytics))
         .app_data(web::Data::new(logs_route))
         .app_data(web::Data::new(logs_stderr))

meilisearch/src/main.rs

@@ -1,8 +1,10 @@
 use std::env;
 use std::io::{stderr, LineWriter, Write};
+use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::thread::available_parallelism;
 
 use actix_web::http::KeepAlive;
 use actix_web::web::Data;
@@ -11,6 +13,7 @@ use index_scheduler::IndexScheduler;
 use is_terminal::IsTerminal;
 use meilisearch::analytics::Analytics;
 use meilisearch::option::LogMode;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{
     analytics, create_app, setup_meilisearch, LogRouteHandle, LogRouteType, LogStderrHandle,
     LogStderrType, Opt, SubscriberForSecondLayer,
@@ -148,11 +151,17 @@ async fn run_http(
     let opt_clone = opt.clone();
     let index_scheduler = Data::from(index_scheduler);
     let auth_controller = Data::from(auth_controller);
+    let search_queue = SearchQueue::new(
+        opt.experimental_search_queue_size,
+        available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
+    );
+    let search_queue = Data::new(search_queue);
 
     let http_server = HttpServer::new(move || {
         create_app(
             index_scheduler.clone(),
             auth_controller.clone(),
+            search_queue.clone(),
             opt.clone(),
             logs.clone(),
             analytics.clone(),

meilisearch/src/middleware.rs

@@ -55,16 +55,17 @@ where
         let index_scheduler = req.app_data::<Data<IndexScheduler>>().unwrap();
         let features = index_scheduler.features();
 
+        let request_path = req.path();
+        let request_pattern = req.match_pattern();
+        let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str).to_string();
+        let request_method = req.method().to_string();
+
         if features.check_metrics().is_ok() {
-            let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
-                let request_pattern = req.match_pattern();
-                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
-                let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, metric_path])
+                        .with_label_values(&[&request_method, &metric_path])
                         .start_timer(),
                 );
             }
@@ -76,11 +77,7 @@ where
             let res = fut.await?;
 
             crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
-                .with_label_values(&[
-                    res.request().method().as_str(),
-                    res.request().path(),
-                    res.status().as_str(),
-                ])
+                .with_label_values(&[&request_method, &metric_path, res.status().as_str()])
                 .inc();
 
             if let Some(histogram_timer) = histogram_timer {

meilisearch/src/routes/indexes/facet_search.rs

@@ -81,7 +81,7 @@ pub async fn search(
     let index = index_scheduler.index(&index_uid)?;
     let features = index_scheduler.features();
     let search_kind = search_kind(&search_query, &index_scheduler, &index, features)?;
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_facet_search(
             &index,
@@ -93,7 +93,9 @@ pub async fn search(
             locales,
         )
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
 
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);

meilisearch/src/routes/indexes/search.rs

@@ -233,11 +233,13 @@ pub async fn search_with_url_query(
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)?;
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_search(&index, query, search_kind, retrieve_vector, index_scheduler.features())
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
@@ -276,11 +278,13 @@ pub async fn search_with_post(
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
 
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_search(&index, query, search_kind, retrieve_vectors, index_scheduler.features())
     })
-    .await?;
+    .await;
+    permit.drop().await;
+    let search_result = search_result?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {

meilisearch/src/routes/indexes/settings.rs

@@ -636,11 +636,19 @@ fn embedder_analytics(
             .any(|config| config.document_template.set().is_some())
     });
 
+    let document_template_max_bytes = setting.as_ref().and_then(|map| {
+        map.values()
+            .filter_map(|config| config.clone().set())
+            .filter_map(|config| config.document_template_max_bytes.set())
+            .max()
+    });
+
     json!(
         {
             "total": setting.as_ref().map(|s| s.len()),
             "sources": sources,
             "document_template_used": document_template_used,
+            "document_template_max_bytes": document_template_max_bytes
         }
     )
 }

meilisearch/src/routes/multi_search.rs

@@ -39,7 +39,7 @@ pub async fn multi_search_with_post(
 ) -> Result<HttpResponse, ResponseError> {
     // Since we don't want to process half of the search requests and then get a permit refused
     // we're going to get one permit for the whole duration of the multi-search request.
-    let _permit = search_queue.try_get_search_permit().await?;
+    let permit = search_queue.try_get_search_permit().await?;
 
     let federated_search = params.into_inner();
 
@@ -81,6 +81,7 @@ pub async fn multi_search_with_post(
                 perform_federated_search(&index_scheduler, queries, federation, features)
             })
             .await;
+            permit.drop().await;
 
             if let Ok(Ok(_)) = search_result {
                 multi_aggregate.succeed();
@@ -143,6 +144,7 @@ pub async fn multi_search_with_post(
                 Ok(search_results)
             }
             .await;
+            permit.drop().await;
 
             if search_results.is_ok() {
                 multi_aggregate.succeed();

meilisearch/src/search/federated.rs

@@ -9,20 +9,24 @@ use std::vec::{IntoIter, Vec};
 
 use actix_http::StatusCode;
 use index_scheduler::{IndexScheduler, RoFeatures};
+use indexmap::IndexMap;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::deserr_codes::{
-    InvalidMultiSearchWeight, InvalidSearchLimit, InvalidSearchOffset,
+    InvalidMultiSearchFacetsByIndex, InvalidMultiSearchMaxValuesPerFacet,
+    InvalidMultiSearchMergeFacets, InvalidMultiSearchWeight, InvalidSearchLimit,
+    InvalidSearchOffset,
 };
 use meilisearch_types::error::ResponseError;
+use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::score_details::{ScoreDetails, ScoreValue};
-use meilisearch_types::milli::{self, DocumentId, TimeBudget};
+use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget};
 use roaring::RoaringBitmap;
 use serde::Serialize;
 
 use super::ranking_rules::{self, RankingRules};
 use super::{
-    prepare_search, AttributesFormat, HitMaker, HitsInfo, RetrieveVectors, SearchHit, SearchKind,
-    SearchQuery, SearchQueryWithIndex,
+    compute_facet_distribution_stats, prepare_search, AttributesFormat, ComputedFacets, FacetStats,
+    HitMaker, HitsInfo, RetrieveVectors, SearchHit, SearchKind, SearchQuery, SearchQueryWithIndex,
 };
 use crate::error::MeilisearchHttpError;
 use crate::routes::indexes::search::search_kind;
@@ -73,6 +77,17 @@ pub struct Federation {
     pub limit: usize,
     #[deserr(default = super::DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
     pub offset: usize,
+    #[deserr(default, error = DeserrJsonError<InvalidMultiSearchFacetsByIndex>)]
+    pub facets_by_index: BTreeMap<IndexUid, Option<Vec<String>>>,
+    #[deserr(default, error = DeserrJsonError<InvalidMultiSearchMergeFacets>)]
+    pub merge_facets: Option<MergeFacets>,
+}
+
+#[derive(Copy, Clone, Debug, deserr::Deserr, Default)]
+#[deserr(error = DeserrJsonError<InvalidMultiSearchMergeFacets>, rename_all = camelCase, deny_unknown_fields)]
+pub struct MergeFacets {
+    #[deserr(default, error = DeserrJsonError<InvalidMultiSearchMaxValuesPerFacet>)]
+    pub max_values_per_facet: Option<usize>,
 }
 
 #[derive(Debug, deserr::Deserr)]
@@ -82,7 +97,7 @@ pub struct FederatedSearch {
     #[deserr(default)]
     pub federation: Option<Federation>,
 }
-#[derive(Serialize, Clone, PartialEq)]
+#[derive(Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct FederatedSearchResult {
     pub hits: Vec<SearchHit>,
@@ -93,6 +108,13 @@ pub struct FederatedSearchResult {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub semantic_hit_count: Option<u32>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub facet_distribution: Option<BTreeMap<String, IndexMap<String, u64>>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub facet_stats: Option<BTreeMap<String, FacetStats>>,
+    #[serde(skip_serializing_if = "FederatedFacets::is_empty")]
+    pub facets_by_index: FederatedFacets,
+
     // These fields are only used for analytics purposes
     #[serde(skip)]
     pub degraded: bool,
@@ -109,6 +131,9 @@ impl fmt::Debug for FederatedSearchResult {
             semantic_hit_count,
             degraded,
             used_negative_operator,
+            facet_distribution,
+            facet_stats,
+            facets_by_index,
         } = self;
 
         let mut debug = f.debug_struct("SearchResult");
@@ -122,9 +147,18 @@ impl fmt::Debug for FederatedSearchResult {
         if *degraded {
             debug.field("degraded", degraded);
         }
+        if let Some(facet_distribution) = facet_distribution {
+            debug.field("facet_distribution", &facet_distribution);
+        }
+        if let Some(facet_stats) = facet_stats {
+            debug.field("facet_stats", &facet_stats);
+        }
         if let Some(semantic_hit_count) = semantic_hit_count {
             debug.field("semantic_hit_count", &semantic_hit_count);
         }
+        if !facets_by_index.is_empty() {
+            debug.field("facets_by_index", &facets_by_index);
+        }
 
         debug.finish()
     }
@@ -313,16 +347,104 @@ struct SearchHitByIndex {
 }
 
 struct SearchResultByIndex {
+    index: String,
     hits: Vec<SearchHitByIndex>,
-    candidates: RoaringBitmap,
+    estimated_total_hits: usize,
     degraded: bool,
     used_negative_operator: bool,
+    facets: Option<ComputedFacets>,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct FederatedFacets(pub BTreeMap<String, ComputedFacets>);
+
+impl FederatedFacets {
+    pub fn insert(&mut self, index: String, facets: Option<ComputedFacets>) {
+        if let Some(facets) = facets {
+            self.0.insert(index, facets);
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn merge(
+        self,
+        MergeFacets { max_values_per_facet }: MergeFacets,
+        facet_order: BTreeMap<String, (String, OrderBy)>,
+    ) -> Option<ComputedFacets> {
+        if self.is_empty() {
+            return None;
+        }
+
+        let mut distribution: BTreeMap<String, _> = Default::default();
+        let mut stats: BTreeMap<String, FacetStats> = Default::default();
+
+        for facets_by_index in self.0.into_values() {
+            for (facet, index_distribution) in facets_by_index.distribution {
+                match distribution.entry(facet) {
+                    std::collections::btree_map::Entry::Vacant(entry) => {
+                        entry.insert(index_distribution);
+                    }
+                    std::collections::btree_map::Entry::Occupied(mut entry) => {
+                        let distribution = entry.get_mut();
+
+                        for (value, index_count) in index_distribution {
+                            distribution
+                                .entry(value)
+                                .and_modify(|count| *count += index_count)
+                                .or_insert(index_count);
+                        }
+                    }
+                }
+            }
+
+            for (facet, index_stats) in facets_by_index.stats {
+                match stats.entry(facet) {
+                    std::collections::btree_map::Entry::Vacant(entry) => {
+                        entry.insert(index_stats);
+                    }
+                    std::collections::btree_map::Entry::Occupied(mut entry) => {
+                        let stats = entry.get_mut();
+
+                        stats.min = f64::min(stats.min, index_stats.min);
+                        stats.max = f64::max(stats.max, index_stats.max);
+                    }
+                }
+            }
+        }
+
+        // fixup order
+        for (facet, values) in &mut distribution {
+            let order_by = facet_order.get(facet).map(|(_, order)| *order).unwrap_or_default();
+
+            match order_by {
+                OrderBy::Lexicographic => {
+                    values.sort_unstable_by(|left, _, right, _| left.cmp(right))
+                }
+                OrderBy::Count => {
+                    values.sort_unstable_by(|_, left, _, right| {
+                        left.cmp(right)
+                            // biggest first
+                            .reverse()
+                    })
+                }
+            }
+
+            if let Some(max_values_per_facet) = max_values_per_facet {
+                values.truncate(max_values_per_facet)
+            };
+        }
+
+        Some(ComputedFacets { distribution, stats })
+    }
 }
 
 pub fn perform_federated_search(
     index_scheduler: &IndexScheduler,
     queries: Vec<SearchQueryWithIndex>,
-    federation: Federation,
+    mut federation: Federation,
     features: RoFeatures,
 ) -> Result<FederatedSearchResult, ResponseError> {
     let before_search = std::time::Instant::now();
@@ -342,6 +464,16 @@ pub fn perform_federated_search(
             .into());
         }
 
+        if let Some(facets) = federated_query.has_facets() {
+            let facets = facets.to_owned();
+            return Err(MeilisearchHttpError::FacetsInFederatedQuery(
+                query_index,
+                federated_query.index_uid.into_inner(),
+                facets,
+            )
+            .into());
+        }
+
         let (index_uid, query, federation_options) = federated_query.into_index_query_federation();
 
         queries_by_index.entry(index_uid.into_inner()).or_default().push(QueryByIndex {
@@ -353,13 +485,24 @@ pub fn perform_federated_search(
 
     // 2. perform queries, merge and make hits index by index
     let required_hit_count = federation.limit + federation.offset;
+
     // In step (2), semantic_hit_count will be set to Some(0) if any search kind uses semantic
     // Then in step (3), we'll update its value if there is any semantic search
     let mut semantic_hit_count = None;
     let mut results_by_index = Vec::with_capacity(queries_by_index.len());
     let mut previous_query_data: Option<(RankingRules, usize, String)> = None;
 
+    // remember the order and name of first index for each facet when merging with index settings
+    // to detect if the order is inconsistent for a facet.
+    let mut facet_order: Option<BTreeMap<String, (String, OrderBy)>> = match federation.merge_facets
+    {
+        Some(MergeFacets { .. }) => Some(Default::default()),
+        _ => None,
+    };
+
     for (index_uid, queries) in queries_by_index {
+        let first_query_index = queries.first().map(|query| query.query_index);
+
         let index = match index_scheduler.index(&index_uid) {
             Ok(index) => index,
             Err(err) => {
@@ -367,9 +510,8 @@ pub fn perform_federated_search(
                 // Patch the HTTP status code to 400 as it defaults to 404 for `index_not_found`, but
                 // here the resource not found is not part of the URL.
                 err.code = StatusCode::BAD_REQUEST;
-                if let Some(query) = queries.first() {
-                    err.message =
-                        format!("Inside `.queries[{}]`: {}", query.query_index, err.message);
+                if let Some(query_index) = first_query_index {
+                    err.message = format!("Inside `.queries[{}]`: {}", query_index, err.message);
                 }
                 return Err(err);
             }
@@ -394,6 +536,23 @@ pub fn perform_federated_search(
         let mut used_negative_operator = false;
         let mut candidates = RoaringBitmap::new();
 
+        let facets_by_index = federation.facets_by_index.remove(&index_uid).flatten();
+
+        // TODO: recover the max size + facets_by_index as return value of this function so as not to ask it for all queries
+        if let Err(mut error) =
+            check_facet_order(&mut facet_order, &index_uid, &facets_by_index, &index, &rtxn)
+        {
+            error.message = format!(
+                "Inside `.federation.facetsByIndex.{index_uid}`: {error}{}",
+                if let Some(query_index) = first_query_index {
+                    format!("\n - Note: index `{index_uid}` used in `.queries[{query_index}]`")
+                } else {
+                    Default::default()
+                }
+            );
+            return Err(error);
+        }
+
         // 2.1. Compute all candidates for each query in the index
         let mut results_by_query = Vec::with_capacity(queries.len());
 
@@ -562,34 +721,116 @@ pub fn perform_federated_search(
                 .collect();
 
         let merged_result = merged_result?;
+
+        let estimated_total_hits = candidates.len() as usize;
+
+        let facets = facets_by_index
+            .map(|facets_by_index| {
+                compute_facet_distribution_stats(
+                    &facets_by_index,
+                    &index,
+                    &rtxn,
+                    candidates,
+                    super::Route::MultiSearch,
+                )
+            })
+            .transpose()
+            .map_err(|mut error| {
+                error.message = format!(
+                    "Inside `.federation.facetsByIndex.{index_uid}`: {}{}",
+                    error.message,
+                    if let Some(query_index) = first_query_index {
+                        format!("\n - Note: index `{index_uid}` used in `.queries[{query_index}]`")
+                    } else {
+                        Default::default()
+                    }
+                );
+                error
+            })?;
+
         results_by_index.push(SearchResultByIndex {
+            index: index_uid,
             hits: merged_result,
-            candidates,
+            estimated_total_hits,
             degraded,
             used_negative_operator,
+            facets,
         });
     }
 
+    // bonus step, make sure to return an error if an index wants a non-faceted field, even if no query actually uses that index.
+    for (index_uid, facets) in federation.facets_by_index {
+        let index = match index_scheduler.index(&index_uid) {
+            Ok(index) => index,
+            Err(err) => {
+                let mut err = ResponseError::from(err);
+                // Patch the HTTP status code to 400 as it defaults to 404 for `index_not_found`, but
+                // here the resource not found is not part of the URL.
+                err.code = StatusCode::BAD_REQUEST;
+                err.message = format!(
+                    "Inside `.federation.facetsByIndex.{index_uid}`: {}\n - Note: index `{index_uid}` is not used in queries",
+                    err.message
+                );
+                return Err(err);
+            }
+        };
+
+        // Important: this is the only transaction we'll use for this index during this federated search
+        let rtxn = index.read_txn()?;
+
+        if let Err(mut error) =
+            check_facet_order(&mut facet_order, &index_uid, &facets, &index, &rtxn)
+        {
+            error.message = format!(
+                "Inside `.federation.facetsByIndex.{index_uid}`: {error}\n - Note: index `{index_uid}` is not used in queries",
+            );
+            return Err(error);
+        }
+
+        if let Some(facets) = facets {
+            if let Err(mut error) = compute_facet_distribution_stats(
+                &facets,
+                &index,
+                &rtxn,
+                Default::default(),
+                super::Route::MultiSearch,
+            ) {
+                error.message =
+                    format!("Inside `.federation.facetsByIndex.{index_uid}`: {}\n - Note: index `{index_uid}` is not used in queries", error.message);
+                return Err(error);
+            }
+        }
+    }
+
     // 3. merge hits and metadata across indexes
     // 3.1 merge metadata
-    let (estimated_total_hits, degraded, used_negative_operator) = {
+    let (estimated_total_hits, degraded, used_negative_operator, facets) = {
         let mut estimated_total_hits = 0;
         let mut degraded = false;
         let mut used_negative_operator = false;
 
+        let mut facets: FederatedFacets = FederatedFacets::default();
+
         for SearchResultByIndex {
+            index,
             hits: _,
-            candidates,
+            estimated_total_hits: estimated_total_hits_by_index,
+            facets: facets_by_index,
             degraded: degraded_by_index,
             used_negative_operator: used_negative_operator_by_index,
-        } in &results_by_index
+        } in &mut results_by_index
         {
-            estimated_total_hits += candidates.len() as usize;
+            estimated_total_hits += *estimated_total_hits_by_index;
             degraded |= *degraded_by_index;
             used_negative_operator |= *used_negative_operator_by_index;
+
+            let facets_by_index = std::mem::take(facets_by_index);
+            let index = std::mem::take(index);
+
+            facets.insert(index, facets_by_index);
         }
 
-        (estimated_total_hits, degraded, used_negative_operator)
+        (estimated_total_hits, degraded, used_negative_operator, facets)
     };
 
     // 3.2 merge hits
@@ -606,6 +847,20 @@ pub fn perform_federated_search(
         .map(|hit| hit.hit)
         .collect();
 
+    let (facet_distribution, facet_stats, facets_by_index) =
+        match federation.merge_facets.zip(facet_order) {
+            Some((merge_facets, facet_order)) => {
+                let facets = facets.merge(merge_facets, facet_order);
+
+                let (facet_distribution, facet_stats) = facets
+                    .map(|ComputedFacets { distribution, stats }| (distribution, stats))
+                    .unzip();
+
+                (facet_distribution, facet_stats, FederatedFacets::default())
+            }
+            None => (None, None, facets),
+        };
+
     let search_result = FederatedSearchResult {
         hits: merged_hits,
         processing_time_ms: before_search.elapsed().as_millis(),
@@ -617,7 +872,39 @@ pub fn perform_federated_search(
         semantic_hit_count,
         degraded,
         used_negative_operator,
+        facet_distribution,
+        facet_stats,
+        facets_by_index,
     };
 
     Ok(search_result)
 }
+
+fn check_facet_order(
+    facet_order: &mut Option<BTreeMap<String, (String, OrderBy)>>,
+    current_index: &str,
+    facets_by_index: &Option<Vec<String>>,
+    index: &milli::Index,
+    rtxn: &milli::heed::RoTxn<'_>,
+) -> Result<(), ResponseError> {
+    if let (Some(facet_order), Some(facets_by_index)) = (facet_order, facets_by_index) {
+        let index_facet_order = index.sort_facet_values_by(rtxn)?;
+        for facet in facets_by_index {
+            let index_facet_order = index_facet_order.get(facet);
+            let (previous_index, previous_facet_order) = facet_order
+                .entry(facet.to_owned())
+                .or_insert_with(|| (current_index.to_owned(), index_facet_order));
+            if previous_facet_order != &index_facet_order {
+                return Err(MeilisearchHttpError::InconsistentFacetOrder {
+                    facet: facet.clone(),
+                    previous_facet_order: *previous_facet_order,
+                    previous_uid: previous_index.clone(),
+                    current_uid: current_index.to_owned(),
+                    index_facet_order,
+                }
+                .into());
+            }
+        }
+    };
+    Ok(())
+}

meilisearch/src/search/mod.rs

@@ -441,9 +441,6 @@ pub struct SearchQueryWithIndex {
 }
 
 impl SearchQueryWithIndex {
-    pub fn has_federation_options(&self) -> bool {
-        self.federation_options.is_some()
-    }
     pub fn has_pagination(&self) -> Option<&'static str> {
         if self.offset.is_some() {
             Some("offset")
@@ -458,6 +455,10 @@ impl SearchQueryWithIndex {
         }
     }
 
+    pub fn has_facets(&self) -> Option<&[String]> {
+        self.facets.as_deref().filter(|v| !v.is_empty())
+    }
+
     pub fn into_index_query_federation(self) -> (IndexUid, SearchQuery, Option<FederationOptions>) {
         let SearchQueryWithIndex {
             index_uid,
@@ -987,39 +988,13 @@ pub fn perform_search(
         HitsInfo::OffsetLimit { limit, offset, estimated_total_hits: number_of_hits }
     };
 
-    let (facet_distribution, facet_stats) = match facets {
-        Some(ref fields) => {
-            let mut facet_distribution = index.facets_distribution(&rtxn);
-
-            let max_values_by_facet = index
-                .max_values_per_facet(&rtxn)
-                .map_err(milli::Error::from)?
-                .map(|x| x as usize)
-                .unwrap_or(DEFAULT_VALUES_PER_FACET);
-            facet_distribution.max_values_per_facet(max_values_by_facet);
-
-            let sort_facet_values_by =
-                index.sort_facet_values_by(&rtxn).map_err(milli::Error::from)?;
-
-            if fields.iter().all(|f| f != "*") {
-                let fields: Vec<_> =
-                    fields.iter().map(|n| (n, sort_facet_values_by.get(n))).collect();
-                facet_distribution.facets(fields);
-            }
-
-            let distribution = facet_distribution
-                .candidates(candidates)
-                .default_order_by(sort_facet_values_by.get("*"))
-                .execute()?;
-            let stats = facet_distribution.compute_stats()?;
-            (Some(distribution), Some(stats))
-        }
-        None => (None, None),
-    };
-
-    let facet_stats = facet_stats.map(|stats| {
-        stats.into_iter().map(|(k, (min, max))| (k, FacetStats { min, max })).collect()
-    });
+    let (facet_distribution, facet_stats) = facets
+        .map(move |facets| {
+            compute_facet_distribution_stats(&facets, index, &rtxn, candidates, Route::Search)
+        })
+        .transpose()?
+        .map(|ComputedFacets { distribution, stats }| (distribution, stats))
+        .unzip();
 
     let result = SearchResult {
         hits: documents,
@@ -1035,6 +1010,61 @@ pub fn perform_search(
     Ok(result)
 }
 
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct ComputedFacets {
+    pub distribution: BTreeMap<String, IndexMap<String, u64>>,
+    pub stats: BTreeMap<String, FacetStats>,
+}
+
+enum Route {
+    Search,
+    MultiSearch,
+}
+
+fn compute_facet_distribution_stats<S: AsRef<str>>(
+    facets: &[S],
+    index: &Index,
+    rtxn: &RoTxn,
+    candidates: roaring::RoaringBitmap,
+    route: Route,
+) -> Result<ComputedFacets, ResponseError> {
+    let mut facet_distribution = index.facets_distribution(rtxn);
+
+    let max_values_by_facet = index
+        .max_values_per_facet(rtxn)
+        .map_err(milli::Error::from)?
+        .map(|x| x as usize)
+        .unwrap_or(DEFAULT_VALUES_PER_FACET);
+
+    facet_distribution.max_values_per_facet(max_values_by_facet);
+
+    let sort_facet_values_by = index.sort_facet_values_by(rtxn).map_err(milli::Error::from)?;
+
+    // add specific facet if there is no placeholder
+    if facets.iter().all(|f| f.as_ref() != "*") {
+        let fields: Vec<_> =
+            facets.iter().map(|n| (n, sort_facet_values_by.get(n.as_ref()))).collect();
+        facet_distribution.facets(fields);
+    }
+
+    let distribution = facet_distribution
+        .candidates(candidates)
+        .default_order_by(sort_facet_values_by.get("*"))
+        .execute()
+        .map_err(|error| match (error, route) {
+            (
+                error @ milli::Error::UserError(milli::UserError::InvalidFacetsDistribution {
+                    ..
+                }),
+                Route::MultiSearch,
+            ) => ResponseError::from_msg(error.to_string(), Code::InvalidMultiSearchFacets),
+            (error, _) => error.into(),
+        })?;
+    let stats = facet_distribution.compute_stats()?;
+    let stats = stats.into_iter().map(|(k, (min, max))| (k, FacetStats { min, max })).collect();
+    Ok(ComputedFacets { distribution, stats })
+}
+
 pub fn search_from_kind(
     search_kind: SearchKind,
     search: milli::Search<'_>,

meilisearch/src/search_queue.rs

@@ -18,6 +18,7 @@
 //!                         And should drop the Permit only once you have freed all the RAM consumed by the method.
 
 use std::num::NonZeroUsize;
+use std::time::Duration;
 
 use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
@@ -29,16 +30,31 @@ use crate::error::MeilisearchHttpError;
 pub struct SearchQueue {
     sender: mpsc::Sender<oneshot::Sender<Permit>>,
     capacity: usize,
+    /// If we have waited longer than this to get a permit, we should abort the search request entirely.
+    /// The client probably already closed the connection, but we have no way to find out.
+    time_to_abort: Duration,
 }
 
 /// You should only run search requests while holding this permit.
 /// Once it's dropped, a new search request will be able to process.
+/// You should always try to drop the permit yourself calling the `drop` async method on it.
 #[derive(Debug)]
 pub struct Permit {
     sender: mpsc::Sender<()>,
 }
 
+impl Permit {
+    /// Drop the permit giving back on permit to the search queue.
+    pub async fn drop(self) {
+        // if the channel is closed then the whole instance is down
+        let _ = self.sender.send(()).await;
+    }
+}
+
 impl Drop for Permit {
+    /// The implicit drop implementation can still be called in multiple cases:
+    /// - We forgot to call the explicit one somewhere => this should be fixed on our side asap
+    /// - The future is cancelled while running and the permit dropped with it
     fn drop(&mut self) {
         let sender = self.sender.clone();
         // if the channel is closed then the whole instance is down
@@ -53,7 +69,11 @@ impl SearchQueue {
         let (sender, receiver) = mpsc::channel(1);
 
         tokio::task::spawn(Self::run(capacity, paralellism, receiver));
-        Self { sender, capacity }
+        Self { sender, capacity, time_to_abort: Duration::from_secs(60) }
+    }
+
+    pub fn with_time_to_abort(self, time_to_abort: Duration) -> Self {
+        Self { time_to_abort, ..self }
     }
 
     /// This function is the main loop, it's in charge on scheduling which search request should execute first and
@@ -119,9 +139,23 @@ impl SearchQueue {
     /// Returns a search `Permit`.
     /// It should be dropped as soon as you've freed all the RAM associated with the search request being processed.
     pub async fn try_get_search_permit(&self) -> Result<Permit, MeilisearchHttpError> {
+        let now = std::time::Instant::now();
         let (sender, receiver) = oneshot::channel();
         self.sender.send(sender).await.map_err(|_| MeilisearchHttpError::SearchLimiterIsDown)?;
-        receiver.await.map_err(|_| MeilisearchHttpError::TooManySearchRequests(self.capacity))
+        let permit = receiver
+            .await
+            .map_err(|_| MeilisearchHttpError::TooManySearchRequests(self.capacity))?;
+
+        // If we've been for more than one minute to get a search permit, it's better to simply
+        // abort the search request than spending time processing something were the client
+        // most certainly exited or got a timeout a long time ago.
+        // We may find a better solution in https://github.com/actix/actix-web/issues/3462.
+        if now.elapsed() > self.time_to_abort {
+            permit.drop().await;
+            Err(MeilisearchHttpError::TooManySearchRequests(self.capacity))
+        } else {
+            Ok(permit)
+        }
     }
 
     /// Returns `Ok(())` if everything seems normal.

meilisearch/tests/auth/mod.rs

@@ -5,63 +5,3 @@ mod payload;
 mod tenant_token;
 
 mod tenant_token_multi_search;
-
-use actix_web::http::StatusCode;
-
-use crate::common::{Server, Value};
-use crate::json;
-
-impl Server {
-    pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {
-        self.service.api_key = Some(api_key.as_ref().to_string());
-    }
-
-    /// Fetch and use the default admin key for nexts http requests.
-    pub async fn use_admin_key(&mut self, master_key: impl AsRef<str>) {
-        self.use_api_key(master_key);
-        let (response, code) = self.list_api_keys("").await;
-        assert_eq!(200, code, "{:?}", response);
-        let admin_key = &response["results"][1]["key"];
-        self.use_api_key(admin_key.as_str().unwrap());
-    }
-
-    pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) {
-        let url = "/keys";
-        self.service.post(url, content).await
-    }
-
-    pub async fn get_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.get(url).await
-    }
-
-    pub async fn patch_api_key(&self, key: impl AsRef<str>, content: Value) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.patch(url, content).await
-    }
-
-    pub async fn list_api_keys(&self, params: &str) -> (Value, StatusCode) {
-        let url = format!("/keys{params}");
-        self.service.get(url).await
-    }
-
-    pub async fn delete_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
-        let url = format!("/keys/{}", key.as_ref());
-        self.service.delete(url).await
-    }
-
-    pub async fn dummy_request(
-        &self,
-        method: impl AsRef<str>,
-        url: impl AsRef<str>,
-    ) -> (Value, StatusCode) {
-        match method.as_ref() {
-            "POST" => self.service.post(url, json!({})).await,
-            "PUT" => self.service.put(url, json!({})).await,
-            "PATCH" => self.service.patch(url, json!({})).await,
-            "GET" => self.service.get(url).await,
-            "DELETE" => self.service.delete(url).await,
-            _ => unreachable!(),
-        }
-    }
-}

meilisearch/tests/auth/tenant_token.rs

@@ -6,7 +6,7 @@ use once_cell::sync::Lazy;
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::{Server, Value};
+use crate::common::{Server, Value, DOCUMENTS};
 use crate::json;
 
 fn generate_tenant_token(
@@ -22,36 +22,6 @@ fn generate_tenant_token(
         .unwrap()
 }
 
-static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Shazam!",
-            "id": "287947",
-            "color": ["green", "blue"]
-        },
-        {
-            "title": "Captain Marvel",
-            "id": "299537",
-            "color": ["yellow", "blue"]
-        },
-        {
-            "title": "Escape Room",
-            "id": "522681",
-            "color": ["yellow", "red"]
-        },
-        {
-            "title": "How to Train Your Dragon: The Hidden World",
-            "id": "166428",
-            "color": ["green", "red"]
-        },
-        {
-            "title": "Glass",
-            "id": "450465",
-            "color": ["blue", "red"]
-        }
-    ])
-});
-
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
     json!({
         "message": null,

meilisearch/tests/common/index.rs

@@ -1,4 +1,5 @@
 use std::fmt::Write;
+use std::marker::PhantomData;
 use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
@@ -9,19 +10,24 @@ use urlencoding::encode as urlencode;
 use super::encoder::Encoder;
 use super::service::Service;
 use super::Value;
+use super::{Owned, Shared};
 use crate::json;
 
-pub struct Index<'a> {
+pub struct Index<'a, State = Owned> {
     pub uid: String,
     pub service: &'a Service,
-    pub encoder: Encoder,
+    pub(super) encoder: Encoder,
+    pub(super) marker: PhantomData<State>,
 }
 
-#[allow(dead_code)]
-impl Index<'_> {
-    pub async fn get(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
-        self.service.get(url).await
+impl<'a> Index<'a, Owned> {
+    pub fn to_shared(&self) -> Index<'a, Shared> {
+        Index {
+            uid: self.uid.clone(),
+            service: self.service,
+            encoder: self.encoder,
+            marker: PhantomData,
+        }
     }
 
     pub async fn load_test_set(&self) -> u64 {
@@ -57,11 +63,7 @@ impl Index<'_> {
     }
 
     pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
-        let body = json!({
-            "uid": self.uid,
-            "primaryKey": primary_key,
-        });
-        self.service.post_encoded("/indexes", body, self.encoder).await
+        self._create(primary_key).await
     }
 
     pub async fn update_raw(&self, body: Value) -> (Value, StatusCode) {
@@ -88,13 +90,7 @@ impl Index<'_> {
         documents: Value,
         primary_key: Option<&str>,
     ) -> (Value, StatusCode) {
-        let url = match primary_key {
-            Some(key) => {
-                format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
-            }
-            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
-        };
-        self.service.post_encoded(url, documents, self.encoder).await
+        self._add_documents(documents, primary_key).await
     }
 
     pub async fn raw_add_documents(
@@ -136,80 +132,11 @@ impl Index<'_> {
         }
     }
 
-    pub async fn wait_task(&self, update_id: u64) -> Value {
-        // try several times to get status, or panic to not wait forever
-        let url = format!("/tasks/{}", update_id);
-        for _ in 0..100 {
-            let (response, status_code) = self.service.get(&url).await;
-            assert_eq!(200, status_code, "response: {}", response);
-
-            if response["status"] == "succeeded" || response["status"] == "failed" {
-                return response;
-            }
-
-            // wait 0.5 second.
-            sleep(Duration::from_millis(500)).await;
-        }
-        panic!("Timeout waiting for update id");
-    }
-
-    pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
-        let url = format!("/tasks/{}", update_id);
-        self.service.get(url).await
-    }
-
     pub async fn list_tasks(&self) -> (Value, StatusCode) {
         let url = format!("/tasks?indexUids={}", self.uid);
         self.service.get(url).await
     }
 
-    pub async fn filtered_tasks(
-        &self,
-        types: &[&str],
-        statuses: &[&str],
-        canceled_by: &[&str],
-    ) -> (Value, StatusCode) {
-        let mut url = format!("/tasks?indexUids={}", self.uid);
-        if !types.is_empty() {
-            let _ = write!(url, "&types={}", types.join(","));
-        }
-        if !statuses.is_empty() {
-            let _ = write!(url, "&statuses={}", statuses.join(","));
-        }
-        if !canceled_by.is_empty() {
-            let _ = write!(url, "&canceledBy={}", canceled_by.join(","));
-        }
-        self.service.get(url).await
-    }
-
-    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
-        if let Some(options) = options {
-            write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
-        }
-        self.service.get(url).await
-    }
-
-    pub async fn get_document_by_filter(&self, payload: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/fetch", urlencode(self.uid.as_ref()));
-        self.service.post(url, payload).await
-    }
-
-    pub async fn get_all_documents_raw(&self, options: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), options);
-        self.service.get(url).await
-    }
-
-    pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let url = format!(
-            "/indexes/{}/documents{}",
-            urlencode(self.uid.as_ref()),
-            yaup::to_string(&options).unwrap()
-        );
-
-        self.service.get(url).await
-    }
-
     pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
         self.service.delete(url).await
@@ -237,14 +164,8 @@ impl Index<'_> {
         self.service.post_encoded(url, body, self.encoder).await
     }
 
-    pub async fn settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.get(url).await
-    }
-
     pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
-        self.service.patch_encoded(url, settings, self.encoder).await
+        self._update_settings(settings).await
     }
 
     pub async fn update_settings_displayed_attributes(
@@ -327,6 +248,146 @@ impl Index<'_> {
         self.service.delete(url).await
     }
 
+    pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
+        let url =
+            format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
+        self.service.put_encoded(url, value, self.encoder).await
+    }
+}
+
+impl<'a> Index<'a, Shared> {
+    /// You cannot modify the content of a shared index, thus the delete_document_by_filter call
+    /// must fail. If the task successfully enqueue itself, we'll wait for the task to finishes,
+    /// and if it succeed the function will panic.
+    pub async fn delete_document_by_filter_fail(&self, body: Value) -> (Value, StatusCode) {
+        let (mut task, code) = self._delete_document_by_filter(body).await;
+        if code.is_success() {
+            task = self.wait_task(task.uid()).await;
+            if task.is_success() {
+                panic!(
+                    "`delete_document_by_filter_fail` succeeded: {}",
+                    serde_json::to_string_pretty(&task).unwrap()
+                );
+            }
+        }
+        (task, code)
+    }
+}
+
+#[allow(dead_code)]
+impl<State> Index<'_, State> {
+    pub async fn get(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
+        self.service.get(url).await
+    }
+
+    /// add_documents is not allowed on shared index but we need to use it to initialize
+    /// a bunch of very common indexes in `common/mod.rs`.
+    pub(super) async fn _add_documents(
+        &self,
+        documents: Value,
+        primary_key: Option<&str>,
+    ) -> (Value, StatusCode) {
+        let url = match primary_key {
+            Some(key) => {
+                format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
+            }
+            None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
+        };
+        self.service.post_encoded(url, documents, self.encoder).await
+    }
+
+    pub(super) async fn _update_settings(&self, settings: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
+        self.service.patch_encoded(url, settings, self.encoder).await
+    }
+
+    pub(super) async fn _delete_document_by_filter(&self, body: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/delete", urlencode(self.uid.as_ref()));
+        self.service.post_encoded(url, body, self.encoder).await
+    }
+
+    pub(super) async fn _create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
+        let body = json!({
+            "uid": self.uid,
+            "primaryKey": primary_key,
+        });
+        self.service.post_encoded("/indexes", body, self.encoder).await
+    }
+    pub async fn wait_task(&self, update_id: u64) -> Value {
+        // try several times to get status, or panic to not wait forever
+        let url = format!("/tasks/{}", update_id);
+        for _ in 0..100 {
+            let (response, status_code) = self.service.get(&url).await;
+            assert_eq!(200, status_code, "response: {}", response);
+
+            if response["status"] == "succeeded" || response["status"] == "failed" {
+                return response;
+            }
+
+            // wait 0.5 second.
+            sleep(Duration::from_millis(500)).await;
+        }
+        panic!("Timeout waiting for update id");
+    }
+
+    pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
+        let url = format!("/tasks/{}", update_id);
+        self.service.get(url).await
+    }
+
+    pub async fn filtered_tasks(
+        &self,
+        types: &[&str],
+        statuses: &[&str],
+        canceled_by: &[&str],
+    ) -> (Value, StatusCode) {
+        let mut url = format!("/tasks?indexUids={}", self.uid);
+        if !types.is_empty() {
+            let _ = write!(url, "&types={}", types.join(","));
+        }
+        if !statuses.is_empty() {
+            let _ = write!(url, "&statuses={}", statuses.join(","));
+        }
+        if !canceled_by.is_empty() {
+            let _ = write!(url, "&canceledBy={}", canceled_by.join(","));
+        }
+        self.service.get(url).await
+    }
+
+    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
+        let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
+        if let Some(options) = options {
+            write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
+        }
+        self.service.get(url).await
+    }
+
+    pub async fn get_document_by_filter(&self, payload: Value) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents/fetch", urlencode(self.uid.as_ref()));
+        self.service.post(url, payload).await
+    }
+
+    pub async fn get_all_documents_raw(&self, options: &str) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), options);
+        self.service.get(url).await
+    }
+
+    pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
+        let url = format!(
+            "/indexes/{}/documents{}",
+            urlencode(self.uid.as_ref()),
+            yaup::to_string(&options).unwrap()
+        );
+
+        self.service.get(url).await
+    }
+
+    pub async fn settings(&self) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
+        self.service.get(url).await
+    }
+
     pub async fn stats(&self) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
         self.service.get(url).await
@@ -411,12 +472,6 @@ impl Index<'_> {
         self.service.post_encoded(url, query, self.encoder).await
     }
 
-    pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
-        let url =
-            format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
-        self.service.put_encoded(url, value, self.encoder).await
-    }
-
     pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
         let url =
             format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");

meilisearch/tests/common/mod.rs

@@ -8,9 +8,16 @@ use std::fmt::{self, Display};
 #[allow(unused)]
 pub use index::GetAllDocumentsOptions;
 use meili_snap::json_string;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 #[allow(unused)]
 pub use server::{default_settings, Server};
+use tokio::sync::OnceCell;
+
+use crate::common::index::Index;
+
+pub enum Shared {}
+pub enum Owned {}
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct Value(pub serde_json::Value);
@@ -27,10 +34,20 @@ impl Value {
         }
     }
 
+    /// Return `true` if the `status` field is set to `succeeded`.
+    /// Panic if the `status` field doesn't exists.
+    #[track_caller]
+    pub fn is_success(&self) -> bool {
+        if !self["status"].is_string() {
+            panic!("Called `is_success` on {}", serde_json::to_string_pretty(&self.0).unwrap());
+        }
+        self["status"] == serde_json::Value::String(String::from("succeeded"))
+    }
+
     // Panic if the json doesn't contain the `status` field set to "succeeded"
     #[track_caller]
     pub fn succeeded(&self) -> &Self {
-        if self["status"] != serde_json::Value::String(String::from("succeeded")) {
+        if !self.is_success() {
             panic!("Called succeeded on {}", serde_json::to_string_pretty(&self.0).unwrap());
         }
         self
@@ -122,3 +139,253 @@ macro_rules! test_post_get_search {
             .map_err(|e| panic!("panic in post route: {:?}", e.downcast_ref::<&str>().unwrap()));
     };
 }
+
+pub async fn shared_does_not_exists_index() -> &'static Index<'static, Shared> {
+    static INDEX: Lazy<Index<'static, Shared>> = Lazy::new(|| {
+        let server = Server::new_shared();
+        server._index("DOES_NOT_EXISTS").to_shared()
+    });
+    &INDEX
+}
+
+pub async fn shared_empty_index() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+
+    INDEX
+        .get_or_init(|| async {
+            let server = Server::new_shared();
+            let index = server._index("EMPTY_INDEX").to_shared();
+            let (response, _code) = index._create(None).await;
+            index.wait_task(response.uid()).await.succeeded();
+            index
+        })
+        .await
+}
+
+pub static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "title": "Shazam!",
+            "id": "287947",
+            "color": ["green", "blue"],
+            "_vectors": { "manual": [1, 2, 3]},
+        },
+        {
+            "title": "Captain Marvel",
+            "id": "299537",
+            "color": ["yellow", "blue"],
+            "_vectors": { "manual": [1, 2, 54] },
+        },
+        {
+            "title": "Escape Room",
+            "id": "522681",
+            "color": ["yellow", "red"],
+            "_vectors": { "manual": [10, -23, 32] },
+        },
+        {
+            "title": "How to Train Your Dragon: The Hidden World",
+            "id": "166428",
+            "color": ["green", "red"],
+            "_vectors": { "manual": [-100, 231, 32] },
+        },
+        {
+            "title": "Gläss",
+            "id": "450465",
+            "color": ["blue", "red"],
+            "_vectors": { "manual": [-100, 340, 90] },
+        }
+    ])
+});
+
+pub async fn shared_index_with_documents() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+    INDEX.get_or_init(|| async {
+        let server = Server::new_shared();
+        let index = server._index("SHARED_DOCUMENTS").to_shared();
+        let documents = DOCUMENTS.clone();
+        let (response, _code) = index._add_documents(documents, None).await;
+        index.wait_task(response.uid()).await.succeeded();
+        let (response, _code) = index
+            ._update_settings(
+                json!({"filterableAttributes": ["id", "title"], "sortableAttributes": ["id", "title"]}),
+            )
+            .await;
+        index.wait_task(response.uid()).await.succeeded();
+        index
+    }).await
+}
+
+pub static SCORE_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "title": "Batman the dark knight returns: Part 1",
+            "id": "A",
+        },
+        {
+            "title": "Batman the dark knight returns: Part 2",
+            "id": "B",
+        },
+        {
+            "title": "Batman Returns",
+            "id": "C",
+        },
+        {
+            "title": "Batman",
+            "id": "D",
+        },
+        {
+            "title": "Badman",
+            "id": "E",
+        }
+    ])
+});
+
+pub static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "id": 852,
+            "father": "jean",
+            "mother": "michelle",
+            "doggos": [
+                {
+                    "name": "bobby",
+                    "age": 2,
+                },
+                {
+                    "name": "buddy",
+                    "age": 4,
+                },
+            ],
+            "cattos": "pésti",
+            "_vectors": { "manual": [1, 2, 3]},
+        },
+        {
+            "id": 654,
+            "father": "pierre",
+            "mother": "sabine",
+            "doggos": [
+                {
+                    "name": "gros bill",
+                    "age": 8,
+                },
+            ],
+            "cattos": ["simba", "pestiféré"],
+            "_vectors": { "manual": [1, 2, 54] },
+        },
+        {
+            "id": 750,
+            "father": "romain",
+            "mother": "michelle",
+            "cattos": ["enigma"],
+            "_vectors": { "manual": [10, 23, 32] },
+        },
+        {
+            "id": 951,
+            "father": "jean-baptiste",
+            "mother": "sophie",
+            "doggos": [
+                {
+                    "name": "turbo",
+                    "age": 5,
+                },
+                {
+                    "name": "fast",
+                    "age": 6,
+                },
+            ],
+            "cattos": ["moumoute", "gomez"],
+            "_vectors": { "manual": [10, 23, 32] },
+        },
+    ])
+});
+
+pub async fn shared_index_with_nested_documents() -> &'static Index<'static, Shared> {
+    static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
+    INDEX.get_or_init(|| async {
+        let server = Server::new_shared();
+        let index = server._index("SHARED_NESTED_DOCUMENTS").to_shared();
+        let documents = NESTED_DOCUMENTS.clone();
+        let (response, _code) = index._add_documents(documents, None).await;
+        index.wait_task(response.uid()).await.succeeded();
+        let (response, _code) = index
+            ._update_settings(
+                json!({"filterableAttributes": ["father", "doggos"], "sortableAttributes": ["doggos"]}),
+            )
+            .await;
+        index.wait_task(response.uid()).await.succeeded();
+        index
+    }).await
+}
+
+pub static FRUITS_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {
+            "name": "Exclusive sale: green apple",
+            "id": "green-apple-boosted",
+            "BOOST": true
+        },
+        {
+            "name": "Pear",
+            "id": "pear",
+        },
+        {
+            "name": "Red apple gala",
+            "id": "red-apple-gala",
+        },
+        {
+            "name": "Exclusive sale: Red Tomato",
+            "id": "red-tomatoes-boosted",
+            "BOOST": true
+        },
+        {
+            "name": "Exclusive sale: Red delicious apple",
+            "id": "red-delicious-boosted",
+            "BOOST": true,
+        }
+    ])
+});
+
+pub static VECTOR_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+      {
+        "id": "A",
+        "description": "the dog barks at the cat",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.9, 0.8, 0.05],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-0.1, 0.55]
+        }
+      },
+      {
+        "id": "B",
+        "description": "the kitten scratched the beagle",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.8, 0.9, 0.5],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-0.2, 0.65]
+        }
+      },
+      {
+        "id": "C",
+        "description": "the dog had to stay alone today",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.85, 0.02, 0.1],
+          // dimensions [negative/positive, energy]
+          "sentiment": [-1.0, 0.1]
+        }
+      },
+      {
+        "id": "D",
+        "description": "the little boy pets the puppy",
+        "_vectors": {
+          // dimensions [canine, feline, young]
+          "animal": [0.8, 0.09, 0.8],
+          // dimensions [negative/positive, energy]
+          "sentiment": [0.8, 0.3]
+        }
+      },
+    ])
+});

meilisearch/tests/common/server.rs

@@ -1,7 +1,8 @@
 #![allow(dead_code)]
 
+use std::marker::PhantomData;
 use std::path::Path;
-use std::str::FromStr as _;
+use std::str::FromStr;
 use std::time::Duration;
 
 use actix_http::body::MessageBody;
@@ -10,28 +11,34 @@ use actix_web::http::StatusCode;
 use byte_unit::{Byte, Unit};
 use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, MaxThreads, Opt};
-use meilisearch::{analytics, create_app, setup_meilisearch, SubscriberForSecondLayer};
+use meilisearch::setup_meilisearch;
 use once_cell::sync::Lazy;
 use tempfile::TempDir;
+use tokio::sync::OnceCell;
 use tokio::time::sleep;
-use tracing::level_filters::LevelFilter;
-use tracing_subscriber::Layer;
+use uuid::Uuid;
 
 use super::index::Index;
 use super::service::Service;
+use super::{Owned, Shared};
 use crate::common::encoder::Encoder;
 use crate::common::Value;
 use crate::json;
 
-pub struct Server {
+pub struct Server<State = Owned> {
     pub service: Service,
     // hold ownership to the tempdir while we use the server instance.
     _dir: Option<TempDir>,
+    _marker: PhantomData<State>,
 }
 
 pub static TEST_TEMP_DIR: Lazy<TempDir> = Lazy::new(|| TempDir::new().unwrap());
 
-impl Server {
+impl Server<Owned> {
+    fn into_shared(self) -> Server<Shared> {
+        Server { service: self.service, _dir: self._dir, _marker: PhantomData }
+    }
+
     pub async fn new() -> Self {
         let dir = TempDir::new().unwrap();
 
@@ -46,7 +53,7 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Server { service, _dir: Some(dir) }
+        Server { service, _dir: Some(dir), _marker: PhantomData }
     }
 
     pub async fn new_auth_with_options(mut options: Opt, dir: TempDir) -> Self {
@@ -61,7 +68,7 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Server { service, _dir: Some(dir) }
+        Server { service, _dir: Some(dir), _marker: PhantomData }
     }
 
     pub async fn new_auth() -> Self {
@@ -74,38 +81,35 @@ impl Server {
         let (index_scheduler, auth) = setup_meilisearch(&options)?;
         let service = Service { index_scheduler, auth, options, api_key: None };
 
-        Ok(Server { service, _dir: None })
+        Ok(Server { service, _dir: None, _marker: PhantomData })
     }
 
-    pub async fn init_web_app(
-        &self,
-    ) -> impl actix_web::dev::Service<
-        actix_http::Request,
-        Response = ServiceResponse<impl MessageBody>,
-        Error = actix_web::Error,
-    > {
-        let (_route_layer, route_layer_handle) =
-            tracing_subscriber::reload::Layer::new(None.with_filter(
-                tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
-            ));
-        let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
-            (Box::new(
-                tracing_subscriber::fmt::layer()
-                    .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
-            )
-                as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
-                .with_filter(tracing_subscriber::filter::Targets::new()),
-        );
+    pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {
+        self.service.api_key = Some(api_key.as_ref().to_string());
+    }
 
-        actix_web::test::init_service(create_app(
-            self.service.index_scheduler.clone().into(),
-            self.service.auth.clone().into(),
-            self.service.options.clone(),
-            (route_layer_handle, stderr_layer_handle),
-            analytics::MockAnalytics::new(&self.service.options),
-            true,
-        ))
-        .await
+    /// Fetch and use the default admin key for nexts http requests.
+    pub async fn use_admin_key(&mut self, master_key: impl AsRef<str>) {
+        self.use_api_key(master_key);
+        let (response, code) = self.list_api_keys("").await;
+        assert_eq!(200, code, "{:?}", response);
+        let admin_key = &response["results"][1]["key"];
+        self.use_api_key(admin_key.as_str().unwrap());
+    }
+
+    pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) {
+        let url = "/keys";
+        self.service.post(url, content).await
+    }
+
+    pub async fn patch_api_key(&self, key: impl AsRef<str>, content: Value) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.patch(url, content).await
+    }
+
+    pub async fn delete_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.delete(url).await
     }
 
     /// Returns a view to an index. There is no guarantee that the index exists.
@@ -118,15 +122,12 @@ impl Server {
     }
 
     pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
-        Index { uid: uid.as_ref().to_string(), service: &self.service, encoder }
-    }
-
-    pub async fn multi_search(&self, queries: Value) -> (Value, StatusCode) {
-        self.service.post("/multi-search", queries).await
-    }
-
-    pub async fn list_indexes_raw(&self, parameters: &str) -> (Value, StatusCode) {
-        self.service.get(format!("/indexes{parameters}")).await
+        Index {
+            uid: uid.as_ref().to_string(),
+            service: &self.service,
+            encoder,
+            marker: PhantomData,
+        }
     }
 
     pub async fn list_indexes(
@@ -150,10 +151,6 @@ impl Server {
         }
     }
 
-    pub async fn version(&self) -> (Value, StatusCode) {
-        self.service.get("/version").await
-    }
-
     pub async fn stats(&self) -> (Value, StatusCode) {
         self.service.get("/stats").await
     }
@@ -162,12 +159,174 @@ impl Server {
         self.service.get("/tasks").await
     }
 
+    pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
+        self.service.patch("/experimental-features", value).await
+    }
+
+    pub async fn get_metrics(&self) -> (Value, StatusCode) {
+        self.service.get("/metrics").await
+    }
+}
+
+impl Server<Shared> {
+    fn init_new_shared_instance() -> Server<Shared> {
+        let dir = TempDir::new().unwrap();
+
+        if cfg!(windows) {
+            std::env::set_var("TMP", TEST_TEMP_DIR.path());
+        } else {
+            std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
+        }
+
+        let options = default_settings(dir.path());
+
+        let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
+        let service = Service { index_scheduler, auth, api_key: None, options };
+
+        Server { service, _dir: Some(dir), _marker: PhantomData }
+    }
+
+    pub fn new_shared() -> &'static Server<Shared> {
+        static SERVER: Lazy<Server<Shared>> = Lazy::new(Server::init_new_shared_instance);
+        &SERVER
+    }
+
+    pub async fn new_shared_with_admin_key() -> &'static Server<Shared> {
+        static SERVER: OnceCell<Server<Shared>> = OnceCell::const_new();
+        SERVER
+            .get_or_init(|| async {
+                let mut server = Server::new_auth().await;
+                server.use_admin_key("MASTER_KEY").await;
+                server.into_shared()
+            })
+            .await
+    }
+
+    /// You shouldn't access random indexes on a shared instance thus this method
+    /// must fail.
+    pub async fn get_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.get(url).await;
+        if code.is_success() {
+            panic!("`get_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    pub async fn delete_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.delete(url).await;
+        if code.is_success() {
+            panic!("`delete_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    pub async fn update_raw_index_fail(
+        &self,
+        uid: impl AsRef<str>,
+        body: Value,
+    ) -> (Value, StatusCode) {
+        let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
+        let (value, code) = self.service.patch_encoded(url, body, Encoder::Plain).await;
+        if code.is_success() {
+            panic!("`update_raw_index_fail` succeeded with uid: {}", uid.as_ref());
+        }
+        (value, code)
+    }
+
+    /// Since this call updates the state of the instance, it must fail.
+    /// If it doesn't fail, the test will panic to help you debug what
+    /// is going on.
+    pub async fn create_index_fail(&self, body: Value) -> (Value, StatusCode) {
+        let (mut task, code) = self._create_index(body).await;
+        if code.is_success() {
+            task = self.wait_task(task.uid()).await;
+            if task.is_success() {
+                panic!(
+                    "`create_index_fail` succeeded: {}",
+                    serde_json::to_string_pretty(&task).unwrap()
+                );
+            }
+        }
+        (task, code)
+    }
+}
+
+impl<State> Server<State> {
+    pub async fn init_web_app(
+        &self,
+    ) -> impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = ServiceResponse<impl MessageBody>,
+        Error = actix_web::Error,
+    > {
+        self.service.init_web_app().await
+    }
+
+    pub async fn list_api_keys(&self, params: &str) -> (Value, StatusCode) {
+        let url = format!("/keys{params}");
+        self.service.get(url).await
+    }
+
+    pub async fn dummy_request(
+        &self,
+        method: impl AsRef<str>,
+        url: impl AsRef<str>,
+    ) -> (Value, StatusCode) {
+        match method.as_ref() {
+            "POST" => self.service.post(url, json!({})).await,
+            "PUT" => self.service.put(url, json!({})).await,
+            "PATCH" => self.service.patch(url, json!({})).await,
+            "GET" => self.service.get(url).await,
+            "DELETE" => self.service.delete(url).await,
+            _ => unreachable!(),
+        }
+    }
+
+    pub async fn get_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
+        let url = format!("/keys/{}", key.as_ref());
+        self.service.get(url).await
+    }
+
+    pub(super) fn _index(&self, uid: impl AsRef<str>) -> Index<'_> {
+        Index {
+            uid: uid.as_ref().to_string(),
+            service: &self.service,
+            encoder: Encoder::Plain,
+            marker: PhantomData,
+        }
+    }
+
+    /// Returns a view to an index. There is no guarantee that the index exists.
+    pub fn unique_index(&self) -> Index<'_> {
+        let uuid = Uuid::new_v4();
+        Index {
+            uid: uuid.to_string(),
+            service: &self.service,
+            encoder: Encoder::Plain,
+            marker: PhantomData,
+        }
+    }
+
+    pub(super) async fn _create_index(&self, body: Value) -> (Value, StatusCode) {
+        self.service.post("/indexes", body).await
+    }
+
+    pub async fn multi_search(&self, queries: Value) -> (Value, StatusCode) {
+        self.service.post("/multi-search", queries).await
+    }
+
+    pub async fn list_indexes_raw(&self, parameters: &str) -> (Value, StatusCode) {
+        self.service.get(format!("/indexes{parameters}")).await
+    }
+
     pub async fn tasks_filter(&self, filter: &str) -> (Value, StatusCode) {
         self.service.get(format!("/tasks?{}", filter)).await
     }
 
-    pub async fn get_dump_status(&self, uid: &str) -> (Value, StatusCode) {
-        self.service.get(format!("/dumps/{}/status", uid)).await
+    pub async fn version(&self) -> (Value, StatusCode) {
+        self.service.get("/version").await
     }
 
     pub async fn create_dump(&self) -> (Value, StatusCode) {
@@ -215,14 +374,6 @@ impl Server {
     pub async fn get_features(&self) -> (Value, StatusCode) {
         self.service.get("/experimental-features").await
     }
-
-    pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
-        self.service.patch("/experimental-features", value).await
-    }
-
-    pub async fn get_metrics(&self) -> (Value, StatusCode) {
-        self.service.get("/metrics").await
-    }
 }
 
 pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
@@ -240,7 +391,8 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
             // memory has to be unlimited because several meilisearch are running in test context.
             max_indexing_memory: MaxMemory::unlimited(),
             skip_index_budget: true,
-            max_indexing_threads: MaxThreads::from_str("1").unwrap(),
+            // Having 2 threads makes the tests way faster
+            max_indexing_threads: MaxThreads::from_str("2").unwrap(),
         },
         experimental_enable_metrics: false,
         ..Parser::parse_from(None as Option<&str>)

meilisearch/tests/common/service.rs

@@ -1,10 +1,15 @@
+use std::num::NonZeroUsize;
 use std::sync::Arc;
 
+use actix_web::body::MessageBody;
+use actix_web::dev::ServiceResponse;
 use actix_web::http::header::ContentType;
 use actix_web::http::StatusCode;
 use actix_web::test;
 use actix_web::test::TestRequest;
+use actix_web::web::Data;
 use index_scheduler::IndexScheduler;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
 use meilisearch_auth::AuthController;
 use tracing::level_filters::LevelFilter;
@@ -106,7 +111,13 @@ impl Service {
         self.request(req).await
     }
 
-    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
+    pub async fn init_web_app(
+        &self,
+    ) -> impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = ServiceResponse<impl MessageBody>,
+        Error = actix_web::Error,
+    > {
         let (_route_layer, route_layer_handle) =
             tracing_subscriber::reload::Layer::new(None.with_filter(
                 tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
@@ -119,16 +130,25 @@ impl Service {
                 as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
                 .with_filter(tracing_subscriber::filter::Targets::new()),
         );
+        let search_queue = SearchQueue::new(
+            self.options.experimental_search_queue_size,
+            NonZeroUsize::new(1).unwrap(),
+        );
 
-        let app = test::init_service(create_app(
+        actix_web::test::init_service(create_app(
             self.index_scheduler.clone().into(),
             self.auth.clone().into(),
+            Data::new(search_queue),
             self.options.clone(),
             (route_layer_handle, stderr_layer_handle),
             analytics::MockAnalytics::new(&self.options),
             true,
         ))
-        .await;
+        .await
+    }
+
+    pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
+        let app = self.init_web_app().await;
 
         if let Some(api_key) = &self.api_key {
             req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));

meilisearch/tests/content_type.rs

@@ -6,6 +6,7 @@ use actix_web::test;
 
 use crate::common::{Server, Value};
 
+#[derive(Debug)]
 enum HttpVerb {
     Put,
     Patch,
@@ -80,7 +81,7 @@ async fn error_json_bad_content_type() {
         let status_code = res.status();
         let body = test::read_body(res).await;
         let response: Value = serde_json::from_slice(&body).unwrap_or_default();
-        assert_eq!(status_code, 415, "calling the route `{}` without content-type is supposed to throw a bad media type error", route);
+        assert_eq!(status_code, 415, "calling the route `{verb:?} {route}` without content-type is supposed to throw a bad media type error:\n{}", String::from_utf8_lossy(&body));
         assert_eq!(
             response,
             json!({

meilisearch/tests/documents/errors.rs

@@ -1,14 +1,15 @@
 use meili_snap::*;
 use urlencoding::encode;
 
-use crate::common::Server;
+use crate::common::{
+    shared_does_not_exists_index, shared_empty_index, shared_index_with_documents, Server,
+};
 use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.get_all_documents_raw("?offset").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -45,9 +46,8 @@ async fn get_all_documents_bad_offset() {
 
 #[actix_rt::test]
 async fn get_all_documents_bad_limit() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.get_all_documents_raw("?limit").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -84,17 +84,13 @@ async fn get_all_documents_bad_limit() {
 
 #[actix_rt::test]
 async fn get_all_documents_bad_filter() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let index = shared_does_not_exists_index().await;
 
-    // Since the filter can't be parsed automatically by deserr, we have the wrong error message
-    // if the index does not exist: we could expect to get an error message about the invalid filter before
-    // the existence of the index is checked, but it is not the case.
     let (response, code) = index.get_all_documents_raw("?filter").await;
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -105,7 +101,7 @@ async fn get_all_documents_bad_filter() {
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -116,42 +112,14 @@ async fn get_all_documents_bad_filter() {
     snapshot!(code, @"404 Not Found");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Index `test` not found.",
+      "message": "Index `DOES_NOT_EXISTS` not found.",
       "code": "index_not_found",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#index_not_found"
     }
     "###);
 
-    let (response, code) = index.create(None).await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 0,
-      "indexUid": "test",
-      "status": "enqueued",
-      "type": "indexCreation",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let response = server.wait_task(0).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }), @r###"
-    {
-      "uid": 0,
-      "indexUid": "test",
-      "status": "succeeded",
-      "type": "indexCreation",
-      "canceledBy": null,
-      "details": {
-        "primaryKey": null
-      },
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
+    let index = shared_empty_index().await;
 
     let (response, code) = index.get_all_documents_raw("?filter").await;
     snapshot!(code, @"200 OK");
@@ -168,7 +136,7 @@ async fn get_all_documents_bad_filter() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `doggo`.\n1:6 doggo",
+      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `doggo`.\n1:6 doggo",
       "code": "invalid_document_filter",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_document_filter"
@@ -189,9 +157,8 @@ async fn get_all_documents_bad_filter() {
 
 #[actix_rt::test]
 async fn delete_documents_batch() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.delete_batch_raw(json!("doggo")).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -206,9 +173,8 @@ async fn delete_documents_batch() {
 
 #[actix_rt::test]
 async fn replace_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_add_documents("", vec![("Content-Type", "application/json")], "").await;
     snapshot!(code, @"400 Bad Request");
@@ -248,9 +214,8 @@ async fn replace_documents_missing_payload() {
 
 #[actix_rt::test]
 async fn update_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", Some("application/json"), "").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -287,9 +252,8 @@ async fn update_documents_missing_payload() {
 
 #[actix_rt::test]
 async fn replace_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_add_documents("", Vec::new(), "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -316,9 +280,8 @@ async fn replace_documents_missing_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", None, "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -345,9 +308,8 @@ async fn update_documents_missing_content_type() {
 
 #[actix_rt::test]
 async fn replace_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_add_documents("", vec![("Content-Type", "doggo")], "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -362,9 +324,8 @@ async fn replace_documents_bad_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.raw_update_documents("", Some("doggo"), "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
@@ -379,9 +340,8 @@ async fn update_documents_bad_content_type() {
 
 #[actix_rt::test]
 async fn replace_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index
         .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter")
         .await;
@@ -428,9 +388,8 @@ async fn replace_documents_bad_csv_delimiter() {
 
 #[actix_rt::test]
 async fn update_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_update_documents("", Some("application/json"), "?csvDelimiter").await;
     snapshot!(code, @"400 Bad Request");
@@ -475,9 +434,8 @@ async fn update_documents_bad_csv_delimiter() {
 
 #[actix_rt::test]
 async fn replace_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index
         .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter=a")
         .await;
@@ -507,9 +465,8 @@ async fn replace_documents_csv_delimiter_with_bad_content_type() {
 
 #[actix_rt::test]
 async fn update_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) =
         index.raw_update_documents("", Some("application/json"), "?csvDelimiter=a").await;
     snapshot!(code, @"415 Unsupported Media Type");
@@ -537,13 +494,12 @@ async fn update_documents_csv_delimiter_with_bad_content_type() {
 
 #[actix_rt::test]
 async fn delete_document_by_filter() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    // send a bad payload type
     let (response, code) = index.delete_document_by_filter(json!("hello")).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type: expected an object, but found a string: `\"hello\"`",
       "code": "bad_request",
@@ -555,7 +511,7 @@ async fn delete_document_by_filter() {
     // send bad payload type
     let (response, code) = index.delete_document_by_filter(json!({ "filter": true })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid syntax for the filter parameter: `expected String, Array, found: true`.",
       "code": "invalid_document_filter",
@@ -567,9 +523,9 @@ async fn delete_document_by_filter() {
     // send bad filter
     let (response, code) = index.delete_document_by_filter(json!({ "filter": "hello"})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
-      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `hello`.\n1:6 hello",
+      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `hello`.\n1:6 hello",
       "code": "invalid_document_filter",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_document_filter"
@@ -579,7 +535,7 @@ async fn delete_document_by_filter() {
     // send empty filter
     let (response, code) = index.delete_document_by_filter(json!({ "filter": ""})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Sending an empty filter is forbidden.",
       "code": "invalid_document_filter",
@@ -591,7 +547,7 @@ async fn delete_document_by_filter() {
     // do not send any filter
     let (response, code) = index.delete_document_by_filter(json!({})).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Missing field `filter`",
       "code": "missing_document_filter",
@@ -600,15 +556,14 @@ async fn delete_document_by_filter() {
     }
     "###);
 
+    let index = shared_does_not_exists_index().await;
     // index does not exists
-    let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "doggo = bernese"})).await;
-    snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let (response, _code) =
+        index.delete_document_by_filter_fail(json!({ "filter": "doggo = bernese"})).await;
+    snapshot!(response, @r###"
     {
-      "uid": 0,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "DOES_NOT_EXISTS",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -618,7 +573,7 @@ async fn delete_document_by_filter() {
         "originalFilter": "\"doggo = bernese\""
       },
       "error": {
-        "message": "Index `doggo` not found.",
+        "message": "Index `DOES_NOT_EXISTS` not found.",
         "code": "index_not_found",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -630,19 +585,14 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (response, code) = index.create(None).await;
-    snapshot!(code, @"202 Accepted");
-    server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-
     // no filterable are set
-    let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "doggo = bernese"})).await;
-    snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let index = shared_empty_index().await;
+    let (response, _code) =
+        index.delete_document_by_filter_fail(json!({ "filter": "doggo = bernese"})).await;
+    snapshot!(response, @r###"
     {
-      "uid": 2,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "EMPTY_INDEX",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -664,19 +614,16 @@ async fn delete_document_by_filter() {
     }
     "###);
 
-    let (response, code) = index.update_settings_filterable_attributes(json!(["doggo"])).await;
-    snapshot!(code, @"202 Accepted");
-    server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-
     // not filterable while there is a filterable attribute
+    let index = shared_index_with_documents().await;
     let (response, code) =
-        index.delete_document_by_filter(json!({ "filter": "catto = jorts"})).await;
+        index.delete_document_by_filter_fail(json!({ "filter": "catto = jorts"})).await;
     snapshot!(code, @"202 Accepted");
-    let response = server.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]"}), @r###"
+    let response = server.wait_task(response.uid()).await;
+    snapshot!(response, @r###"
     {
-      "uid": 4,
-      "indexUid": "doggo",
+      "uid": "[uid]",
+      "indexUid": "SHARED_DOCUMENTS",
       "status": "failed",
       "type": "documentDeletion",
       "canceledBy": null,
@@ -686,7 +633,7 @@ async fn delete_document_by_filter() {
         "originalFilter": "\"catto = jorts\""
       },
       "error": {
-        "message": "Attribute `catto` is not filterable. Available filterable attributes are: `doggo`.\n1:6 catto = jorts",
+        "message": "Attribute `catto` is not filterable. Available filterable attributes are: `id`, `title`.\n1:6 catto = jorts",
         "code": "invalid_document_filter",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_document_filter"
@@ -701,10 +648,10 @@ async fn delete_document_by_filter() {
 
 #[actix_rt::test]
 async fn fetch_document_by_filter() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     index.update_settings_filterable_attributes(json!(["color"])).await;
-    index
+    let (task, _code) = index
         .add_documents(
             json!([
                 { "id": 0, "color": "red" },
@@ -715,7 +662,7 @@ async fn fetch_document_by_filter() {
             Some("id"),
         )
         .await;
-    index.wait_task(1).await;
+    index.wait_task(task.uid()).await.succeeded();
 
     let (response, code) = index.get_document_by_filter(json!(null)).await;
     snapshot!(code, @"400 Bad Request");
@@ -776,7 +723,7 @@ async fn fetch_document_by_filter() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(response, @r###"
     {
-      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `cool doggo`.\n1:11 cool doggo",
+      "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `cool doggo`.\n1:11 cool doggo",
       "code": "invalid_document_filter",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_document_filter"
@@ -798,8 +745,7 @@ async fn fetch_document_by_filter() {
 
 #[actix_rt::test]
 async fn retrieve_vectors() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let index = shared_empty_index().await;
 
     // GET ALL DOCUMENTS BY QUERY
     let (response, _code) = index.get_all_documents_raw("?retrieveVectors=tamo").await;

meilisearch/tests/dumps/mod.rs

@@ -2097,7 +2097,8 @@ async fn generate_and_import_dump_containing_vectors() {
           "source": "huggingFace",
           "model": "sentence-transformers/all-MiniLM-L6-v2",
           "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
-          "documentTemplate": "{{doc.doggo}}"
+          "documentTemplate": "{{doc.doggo}}",
+          "documentTemplateMaxBytes": 400
         }
       },
       "searchCutoffMs": null,

meilisearch/tests/index/errors.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?offset=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -21,7 +21,7 @@ async fn get_indexes_bad_offset() {
 
 #[actix_rt::test]
 async fn get_indexes_bad_limit() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?limit=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -37,7 +37,7 @@ async fn get_indexes_bad_limit() {
 
 #[actix_rt::test]
 async fn get_indexes_unknown_field() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.list_indexes_raw("?doggo=nolimit").await;
     snapshot!(code, @"400 Bad Request");
@@ -53,9 +53,9 @@ async fn get_indexes_unknown_field() {
 
 #[actix_rt::test]
 async fn create_index_missing_uid() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "primaryKey": "doggo" })).await;
+    let (response, code) = server.create_index_fail(json!({ "primaryKey": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -69,9 +69,9 @@ async fn create_index_missing_uid() {
 
 #[actix_rt::test]
 async fn create_index_bad_uid() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "uid": "the best doggo" })).await;
+    let (response, code) = server.create_index_fail(json!({ "uid": "the best doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -82,7 +82,7 @@ async fn create_index_bad_uid() {
     }
     "###);
 
-    let (response, code) = server.create_index(json!({ "uid": true })).await;
+    let (response, code) = server.create_index_fail(json!({ "uid": true })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -96,10 +96,10 @@ async fn create_index_bad_uid() {
 
 #[actix_rt::test]
 async fn create_index_bad_primary_key() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server
-        .create_index(json!({ "uid": "doggo", "primaryKey": ["the", "best", "doggo"] }))
+        .create_index_fail(json!({ "uid": "doggo", "primaryKey": ["the", "best", "doggo"] }))
         .await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -114,9 +114,10 @@ async fn create_index_bad_primary_key() {
 
 #[actix_rt::test]
 async fn create_index_unknown_field() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
-    let (response, code) = server.create_index(json!({ "uid": "doggo", "doggo": "bernese" })).await;
+    let (response, code) =
+        server.create_index_fail(json!({ "uid": "doggo", "doggo": "bernese" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -130,10 +131,8 @@ async fn create_index_unknown_field() {
 
 #[actix_rt::test]
 async fn get_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.get().await;
+    let server = Server::new_shared();
+    let (response, code) = server.get_index_fail("the good doggo").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -147,9 +146,8 @@ async fn get_index_bad_uid() {
 
 #[actix_rt::test]
 async fn update_index_bad_primary_key() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "primaryKey": ["doggo"] })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -164,9 +162,8 @@ async fn update_index_bad_primary_key() {
 
 #[actix_rt::test]
 async fn update_index_immutable_uid() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "uid": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -181,9 +178,8 @@ async fn update_index_immutable_uid() {
 
 #[actix_rt::test]
 async fn update_index_immutable_created_at() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "createdAt": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -198,9 +194,8 @@ async fn update_index_immutable_created_at() {
 
 #[actix_rt::test]
 async fn update_index_immutable_updated_at() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "updatedAt": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -215,9 +210,8 @@ async fn update_index_immutable_updated_at() {
 
 #[actix_rt::test]
 async fn update_index_unknown_field() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (response, code) = index.update_raw(json!({ "doggo": "bork" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -232,10 +226,9 @@ async fn update_index_unknown_field() {
 
 #[actix_rt::test]
 async fn update_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.update_raw(json!({ "primaryKey": "doggo" })).await;
+    let server = Server::new_shared();
+    let (response, code) =
+        server.update_raw_index_fail("the good doggo", json!({ "primaryKey": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -249,10 +242,8 @@ async fn update_index_bad_uid() {
 
 #[actix_rt::test]
 async fn delete_index_bad_uid() {
-    let server = Server::new().await;
-    let index = server.index("the good doggo");
-
-    let (response, code) = index.delete().await;
+    let server = Server::new_shared();
+    let (response, code) = server.delete_index_fail("the good doggo").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {

meilisearch/tests/logs/error.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn logs_stream_bad_target() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) = server.service.post("/logs/stream", json!({ "target": true })).await;
@@ -58,7 +58,7 @@ async fn logs_stream_bad_target() {
 
 #[actix_rt::test]
 async fn logs_stream_bad_mode() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) = server.service.post("/logs/stream", json!({ "mode": true })).await;
@@ -99,7 +99,7 @@ async fn logs_stream_bad_mode() {
 
 #[actix_rt::test]
 async fn logs_stream_bad_profile_memory() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     // Wrong type
     let (response, code) =
@@ -156,7 +156,7 @@ async fn logs_stream_bad_profile_memory() {
 
 #[actix_rt::test]
 async fn logs_stream_without_enabling_the_route() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.service.post("/logs/stream", json!({})).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/logs/mod.rs

@@ -1,10 +1,13 @@
 mod error;
 
+use std::num::NonZeroUsize;
 use std::rc::Rc;
 use std::str::FromStr;
 
 use actix_web::http::header::ContentType;
+use actix_web::web::Data;
 use meili_snap::snapshot;
+use meilisearch::search_queue::SearchQueue;
 use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
 use tracing::level_filters::LevelFilter;
 use tracing_subscriber::layer::SubscriberExt;
@@ -40,10 +43,15 @@ async fn basic_test_log_stream_route() {
             .with_span_events(tracing_subscriber::fmt::format::FmtSpan::ACTIVE)
             .with_filter(tracing_subscriber::filter::LevelFilter::from_str("OFF").unwrap()),
     );
+    let search_queue = SearchQueue::new(
+        server.service.options.experimental_search_queue_size,
+        NonZeroUsize::new(1).unwrap(),
+    );
 
     let app = actix_web::test::init_service(create_app(
         server.service.index_scheduler.clone().into(),
         server.service.auth.clone().into(),
+        Data::new(search_queue),
         server.service.options.clone(),
         (route_layer_handle, stderr_layer_handle),
         analytics::MockAnalytics::new(&server.service.options),

meilisearch/tests/search/errors.rs

@@ -1,16 +1,13 @@
 use meili_snap::*;
 
-use super::DOCUMENTS;
-use crate::common::Server;
+use crate::common::{shared_does_not_exists_index, Server};
 use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
+    let index = shared_does_not_exists_index().await;
     let expected_response = json!({
-        "message": "Index `test` not found.",
+        "message": "Index `DOES_NOT_EXISTS` not found.",
         "code": "index_not_found",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#index_not_found"
@@ -26,8 +23,8 @@ async fn search_unexisting_index() {
 
 #[actix_rt::test]
 async fn search_unexisting_parameter() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     index
         .search(json!({"marin": "hello"}), |response, code| {
@@ -39,8 +36,8 @@ async fn search_unexisting_parameter() {
 
 #[actix_rt::test]
 async fn search_bad_q() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"q": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -57,8 +54,8 @@ async fn search_bad_q() {
 
 #[actix_rt::test]
 async fn search_bad_offset() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"offset": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -85,8 +82,8 @@ async fn search_bad_offset() {
 
 #[actix_rt::test]
 async fn search_bad_limit() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"limit": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -113,8 +110,8 @@ async fn search_bad_limit() {
 
 #[actix_rt::test]
 async fn search_bad_page() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"page": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -141,8 +138,8 @@ async fn search_bad_page() {
 
 #[actix_rt::test]
 async fn search_bad_hits_per_page() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"hitsPerPage": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -169,8 +166,8 @@ async fn search_bad_hits_per_page() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_retrieve() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToRetrieve": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -187,8 +184,8 @@ async fn search_bad_attributes_to_retrieve() {
 
 #[actix_rt::test]
 async fn search_bad_retrieve_vectors() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"retrieveVectors": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -237,8 +234,8 @@ async fn search_bad_retrieve_vectors() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_crop() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToCrop": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -255,8 +252,8 @@ async fn search_bad_attributes_to_crop() {
 
 #[actix_rt::test]
 async fn search_bad_crop_length() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"cropLength": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -283,8 +280,8 @@ async fn search_bad_crop_length() {
 
 #[actix_rt::test]
 async fn search_bad_attributes_to_highlight() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"attributesToHighlight": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -303,14 +300,12 @@ async fn search_bad_attributes_to_highlight() {
 async fn search_bad_filter() {
     // Since a filter is deserialized as a json Value it will never fail to deserialize.
     // Thus the error message is not generated by deserr but written by us.
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     // Also, to trigger the error message we need to effectively create the index or else it'll throw an
     // index does not exists error.
-    let (_, code) = index.create(None).await;
-    server.wait_task(0).await;
-
-    snapshot!(code, @"202 Accepted");
+    let (response, _code) = index.create(None).await;
+    server.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({ "filter": true })).await;
     snapshot!(code, @"400 Bad Request");
@@ -327,8 +322,8 @@ async fn search_bad_filter() {
 
 #[actix_rt::test]
 async fn search_bad_sort() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"sort": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -345,8 +340,8 @@ async fn search_bad_sort() {
 
 #[actix_rt::test]
 async fn search_bad_show_matches_position() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"showMatchesPosition": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -373,8 +368,8 @@ async fn search_bad_show_matches_position() {
 
 #[actix_rt::test]
 async fn search_bad_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"facets": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -391,8 +386,8 @@ async fn search_bad_facets() {
 
 #[actix_rt::test]
 async fn search_bad_threshold() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"rankingScoreThreshold": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -408,8 +403,8 @@ async fn search_bad_threshold() {
 
 #[actix_rt::test]
 async fn search_invalid_threshold() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"rankingScoreThreshold": 42})).await;
     snapshot!(code, @"400 Bad Request");
@@ -425,11 +420,11 @@ async fn search_invalid_threshold() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     // Wait for the settings update to complete
-    index.wait_task(0).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -456,10 +451,11 @@ async fn search_non_filterable_facets() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_multiple_filterable() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -486,10 +482,10 @@ async fn search_non_filterable_facets_multiple_filterable() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_no_filterable() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": []})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"filterableAttributes": []})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -516,10 +512,11 @@ async fn search_non_filterable_facets_no_filterable() {
 
 #[actix_rt::test]
 async fn search_non_filterable_facets_multiple_facets() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
-    index.wait_task(0).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _uid) =
+        index.update_settings(json!({"filterableAttributes": ["title", "genres"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let (response, code) = index.search_post(json!({"facets": ["doggo", "neko"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -546,8 +543,8 @@ async fn search_non_filterable_facets_multiple_facets() {
 
 #[actix_rt::test]
 async fn search_bad_highlight_pre_tag() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"highlightPreTag": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -564,8 +561,8 @@ async fn search_bad_highlight_pre_tag() {
 
 #[actix_rt::test]
 async fn search_bad_highlight_post_tag() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"highlightPostTag": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -582,8 +579,8 @@ async fn search_bad_highlight_post_tag() {
 
 #[actix_rt::test]
 async fn search_bad_crop_marker() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"cropMarker": ["doggo"]})).await;
     snapshot!(code, @"400 Bad Request");
@@ -600,8 +597,8 @@ async fn search_bad_crop_marker() {
 
 #[actix_rt::test]
 async fn search_bad_matching_strategy() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.search_post(json!({"matchingStrategy": "doggo"})).await;
     snapshot!(code, @"400 Bad Request");
@@ -639,20 +636,17 @@ async fn search_bad_matching_strategy() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_object() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    let (task, _code) = index.add_documents(documents, None).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     index.wait_task(task.uid()).await;
 
     index
         .search(json!({"filter": "title & Glass"}), |response, code| {
             snapshot!(response, @r###"
             {
-              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
+              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
               "code": "invalid_search_filter",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_search_filter"
@@ -665,20 +659,17 @@ async fn filter_invalid_syntax_object() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    let (task, _code) = index.add_documents(documents, None).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
     index.wait_task(task.uid()).await;
 
     index
         .search(json!({"filter": ["title & Glass"]}), |response, code| {
             snapshot!(response, @r###"
             {
-              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
+              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
               "code": "invalid_search_filter",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_search_filter"
@@ -691,14 +682,11 @@ async fn filter_invalid_syntax_array() {
 
 #[actix_rt::test]
 async fn filter_invalid_syntax_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Found unexpected characters at the end of the filter: `XOR title = Glass`. You probably forgot an `OR` or an `AND` rule.\n15:32 title = Glass XOR title = Glass",
@@ -716,14 +704,11 @@ async fn filter_invalid_syntax_string() {
 
 #[actix_rt::test]
 async fn filter_invalid_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.\n1:5 many = Glass",
@@ -741,14 +726,11 @@ async fn filter_invalid_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_invalid_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.\n1:5 many = Glass",
@@ -766,14 +748,11 @@ async fn filter_invalid_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:13 _geo = Glass",
@@ -791,14 +770,11 @@ async fn filter_reserved_geo_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:13 _geo = Glass",
@@ -816,14 +792,11 @@ async fn filter_reserved_geo_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_attribute_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:21 _geoDistance = Glass",
@@ -841,14 +814,11 @@ async fn filter_reserved_attribute_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_attribute_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
        "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:21 _geoDistance = Glass",
@@ -866,14 +836,11 @@ async fn filter_reserved_attribute_string() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_point_array() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:18 _geoPoint = Glass",
@@ -891,14 +858,11 @@ async fn filter_reserved_geo_point_array() {
 
 #[actix_rt::test]
 async fn filter_reserved_geo_point_string() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"filterableAttributes": ["title"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
        "message": "`_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.\n1:18 _geoPoint = Glass",
@@ -916,14 +880,11 @@ async fn filter_reserved_geo_point_string() {
 
 #[actix_rt::test]
 async fn sort_geo_reserved_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geo` is a reserved keyword and thus can't be used as a sort expression. Use the _geoPoint(latitude, longitude) built-in rule to sort on _geo field coordinates.",
@@ -946,14 +907,11 @@ async fn sort_geo_reserved_attribute() {
 
 #[actix_rt::test]
 async fn sort_reserved_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (task, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(task.uid()).await;
 
     let expected_response = json!({
         "message": "`_geoDistance` is a reserved keyword and thus can't be used as a sort expression.",
@@ -976,14 +934,10 @@ async fn sort_reserved_attribute() {
 
 #[actix_rt::test]
 async fn sort_unsortable_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "Attribute `title` is not sortable. Available sortable attributes are: `id`.",
@@ -1006,14 +960,11 @@ async fn sort_unsortable_attribute() {
 
 #[actix_rt::test]
 async fn sort_invalid_syntax() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let (response, _code) = index.update_settings(json!({"sortableAttributes": ["id"]})).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "Invalid syntax for the sort parameter: expected expression ending by `:asc` or `:desc`, found `title`.",
@@ -1036,18 +987,15 @@ async fn sort_invalid_syntax() {
 
 #[actix_rt::test]
 async fn sort_unset_ranking_rule() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
-    index
+    let (response, _code) = index
         .update_settings(
             json!({"sortableAttributes": ["title"], "rankingRules": ["proximity", "exactness"]}),
         )
         .await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     let expected_response = json!({
         "message": "You must specify where `sort` is listed in the rankingRules setting to use the sort parameter at search time.",
@@ -1070,14 +1018,11 @@ async fn sort_unset_ranking_rule() {
 
 #[actix_rt::test]
 async fn search_on_unknown_field() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings_searchable_attributes(json!(["id", "title"])).await;
-    index.wait_task(0).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings_searchable_attributes(json!(["id", "title"])).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     index
         .search(
@@ -1099,14 +1044,11 @@ async fn search_on_unknown_field() {
 
 #[actix_rt::test]
 async fn search_on_unknown_field_plus_joker() {
-    let server = Server::new().await;
-    let index = server.index("test");
-    index.update_settings_searchable_attributes(json!(["id", "title"])).await;
-    index.wait_task(0).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
+    let server = Server::new_shared();
+    let index = server.unique_index();
+    let (response, _code) =
+        index.update_settings_searchable_attributes(json!(["id", "title"])).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     index
         .search(
@@ -1145,11 +1087,10 @@ async fn search_on_unknown_field_plus_joker() {
 
 #[actix_rt::test]
 async fn distinct_at_search_time() {
-    let server = Server::new().await;
-    let index = server.index("tamo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     let (task, _) = index.create(None).await;
-    let task = index.wait_task(task.uid()).await;
-    snapshot!(task, name: "task-succeed");
+    index.wait_task(task.uid()).await.succeeded();
 
     let (response, code) =
         index.search_post(json!({"page": 0, "hitsPerPage": 2, "distinct": "doggo.truc"})).await;
@@ -1210,8 +1151,8 @@ async fn distinct_at_search_time() {
 async fn search_with_contains_without_enabling_the_feature() {
     // Since a filter is deserialized as a json Value it will never fail to deserialize.
     // Thus the error message is not generated by deserr but written by us.
-    let server = Server::new().await;
-    let index = server.index("doggo");
+    let server = Server::new_shared();
+    let index = server.unique_index();
     // Also, to trigger the error message we need to effectively create the index or else it'll throw an
     // index does not exists error.
     let (task, _code) = index.create(None).await;
@@ -1222,7 +1163,7 @@ async fn search_with_contains_without_enabling_the_feature() {
             snapshot!(code, @"400 Bad Request");
             snapshot!(json_string!(response), @r###"
             {
-              "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
+              "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
               "code": "feature_not_enabled",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
@@ -1235,7 +1176,7 @@ async fn search_with_contains_without_enabling_the_feature() {
             snapshot!(code, @"400 Bad Request");
             snapshot!(json_string!(response), @r###"
             {
-              "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n25:33 doggo != echo AND doggo CONTAINS kefir",
+              "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n25:33 doggo != echo AND doggo CONTAINS kefir",
               "code": "feature_not_enabled",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
@@ -1251,7 +1192,7 @@ async fn search_with_contains_without_enabling_the_feature() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
+      "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
       "code": "feature_not_enabled",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
@@ -1263,7 +1204,7 @@ async fn search_with_contains_without_enabling_the_feature() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
+      "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir",
       "code": "feature_not_enabled",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#feature_not_enabled"

meilisearch/tests/search/locales.rs

@@ -103,41 +103,12 @@ async fn simple_search() {
 
     // english
     index
-        .search(json!({"q": "Atta", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "Atta", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_en": "Attack on Titan",
-                  "name_ja": "進撃の巨人",
-                  "author_en": "Hajime Isayama",
-                  "author_ja": "諫山 創",
-                  "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                  "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                  "id": 852,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_en": "<em>Atta</em>ck on Titan",
-                    "name_ja": "進撃の巨人",
-                    "author_en": "Hajime Isayama",
-                    "author_ja": "諫山 創",
-                    "description_en": "<em>Atta</em>ck on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                    "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                    "id": "852",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
+                  "id": 852
                 }
               ],
               "query": "Atta",
@@ -153,35 +124,12 @@ async fn simple_search() {
 
     // japanese
     index
-        .search(json!({"q": "進撃", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "進撃", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_zh": "进击的巨人",
-                  "author_zh": "諫山創",
-                  "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                  "id": 853,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_zh": "<em>进击</em>的巨人",
-                    "author_zh": "諫山創",
-                    "description_zh": "<em>进击</em>的巨人是日本的漫画系列，由諫山 創作画。",
-                    "id": "853",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
+                  "id": 853
                 }
               ],
               "query": "進撃",
@@ -197,68 +145,16 @@ async fn simple_search() {
 
     index
         .search(
-            json!({"q": "進撃", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "進撃", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_en": "Attack on Titan",
-                      "name_ja": "進撃の巨人",
-                      "author_en": "Hajime Isayama",
-                      "author_ja": "諫山 創",
-                      "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                      "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_en": "Attack on Titan",
-                        "name_ja": "<em>進撃</em>の巨人",
-                        "author_en": "Hajime Isayama",
-                        "author_ja": "諫山 創",
-                        "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 852
                     },
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "进击的巨人",
-                        "author_zh": "諫山創",
-                        "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "進撃",
@@ -275,67 +171,15 @@ async fn simple_search() {
 
     // chinese
     index
-        .search(json!({"q": "进击", "attributesToHighlight": ["*"]}), |response, code| {
+        .search(json!({"q": "进击", "attributesToRetrieve": ["id"]}), |response, code| {
             snapshot!(response, @r###"
             {
               "hits": [
                 {
-                  "name_zh": "进击的巨人",
-                  "author_zh": "諫山創",
-                  "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                  "id": 853,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_zh": "<em>进击</em>的巨人",
-                    "author_zh": "諫山創",
-                    "description_zh": "<em>进击</em>的巨人是日本的漫画系列，由諫山 創作画。",
-                    "id": "853",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
+                  "id": 853
                 },
                 {
-                  "name_en": "Attack on Titan",
-                  "name_ja": "進撃の巨人",
-                  "author_en": "Hajime Isayama",
-                  "author_ja": "諫山 創",
-                  "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                  "description_ja": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                  "id": 852,
-                  "_vectors": {
-                    "manual": [
-                      1.0,
-                      2.0,
-                      3.0
-                    ]
-                  },
-                  "_formatted": {
-                    "name_en": "Attack on Titan",
-                    "name_ja": "<em>進撃</em>の巨人",
-                    "author_en": "Hajime Isayama",
-                    "author_ja": "諫山 創",
-                    "description_en": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                    "description_ja": "<em>進撃</em>の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                    "id": "852",
-                    "_vectors": {
-                      "manual": [
-                        "1.0",
-                        "2.0",
-                        "3.0"
-                      ]
-                    }
-                  }
+                  "id": 852
                 }
               ],
               "query": "进击",
@@ -382,36 +226,13 @@ async fn force_locales() {
     // chinese detection
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -429,36 +250,13 @@ async fn force_locales() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -506,36 +304,13 @@ async fn force_locales_with_pattern() {
     // chinese detection
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -553,36 +328,13 @@ async fn force_locales_with_pattern() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -628,7 +380,7 @@ async fn force_locales_with_pattern_nested() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -648,60 +400,13 @@ async fn force_locales_with_pattern_nested() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "document_en": {
-                        "name": "Attack on Titan",
-                        "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "author": "Hajime Isayama"
-                      },
-                      "document_ja": {
-                        "name": "進撃の巨人",
-                        "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "author": "諫山 創"
-                      },
-                      "document_zh": {
-                        "name": "进击的巨人",
-                        "description": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "author": "諫山創"
-                      },
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "document_en": {
-                          "name": "Attack on Titan",
-                          "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                          "author": "Hajime Isayama"
-                        },
-                        "document_ja": {
-                          "name": "進撃の巨人",
-                          "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                          "author": "諫山 創"
-                        },
-                        "document_zh": {
-                          "name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                          "description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                          "author": "諫山創"
-                        },
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 852
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -750,7 +455,7 @@ async fn force_different_locales_with_pattern() {
     // force chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -770,36 +475,13 @@ async fn force_different_locales_with_pattern() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -851,7 +533,7 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
     // auto infer any language
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -871,36 +553,13 @@ async fn auto_infer_locales_at_search_with_attributes_to_search_on() {
     // should infer chinese
     index
             .search(
-                json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"], "attributesToSearchOn": ["name_zh", "description_zh"]}),
+                json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"], "attributesToSearchOn": ["name_zh", "description_zh"]}),
                 |response, code| {
                     snapshot!(response, @r###"
                     {
                       "hits": [
                         {
-                          "name_zh": "进击的巨人",
-                          "author_zh": "諫山創",
-                          "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                          "id": 853,
-                          "_vectors": {
-                            "manual": [
-                              1.0,
-                              2.0,
-                              3.0
-                            ]
-                          },
-                          "_formatted": {
-                            "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                            "author_zh": "諫山創",
-                            "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                            "id": "853",
-                            "_vectors": {
-                              "manual": [
-                                "1.0",
-                                "2.0",
-                                "3.0"
-                              ]
-                            }
-                          }
+                          "id": 853
                         }
                       ],
                       "query": "\"进击的巨人\"",
@@ -947,36 +606,13 @@ async fn auto_infer_locales_at_search() {
 
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -992,37 +628,14 @@ async fn auto_infer_locales_at_search() {
         .await;
 
     index
-            .search(
-                json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
-                |response, code| {
-                    snapshot!(response, @r###"
+        .search(
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
+            |response, code| {
+                snapshot!(response, @r###"
                     {
                       "hits": [
                         {
-                          "name_zh": "进击的巨人",
-                          "author_zh": "諫山創",
-                          "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                          "id": 853,
-                          "_vectors": {
-                            "manual": [
-                              1.0,
-                              2.0,
-                              3.0
-                            ]
-                          },
-                          "_formatted": {
-                            "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                            "author_zh": "諫山創",
-                            "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                            "id": "853",
-                            "_vectors": {
-                              "manual": [
-                                "1.0",
-                                "2.0",
-                                "3.0"
-                              ]
-                            }
-                          }
+                          "id": 853
                         }
                       ],
                       "query": "\"进击的巨人\"",
@@ -1032,43 +645,20 @@ async fn auto_infer_locales_at_search() {
                       "estimatedTotalHits": 1
                     }
                     "###);
-                    snapshot!(code, @"200 OK");
-                },
-            )
-            .await;
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
 
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "name_zh": "进击的巨人",
-                      "author_zh": "諫山創",
-                      "description_zh": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                      "id": 853,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "name_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                        "author_zh": "諫山創",
-                        "description_zh": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                        "id": "853",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 853
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -1116,7 +706,7 @@ async fn force_different_locales_with_pattern_nested() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1136,60 +726,37 @@ async fn force_different_locales_with_pattern_nested() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
+            |response, code| {
+                snapshot!(response, @r###"
+                    {
+                      "hits": [
+                        {
+                          "id": 852
+                        }
+                      ],
+                      "query": "\"进击的巨人\"",
+                      "processingTimeMs": "[duration]",
+                      "limit": 20,
+                      "offset": 0,
+                      "estimatedTotalHits": 1
+                    }
+                    "###);
+                snapshot!(code, @"200 OK");
+            },
+        )
+        .await;
+
+    // force japanese
+    index
+        .search(
+            json!({"q": "\"进击的巨人\"", "locales": ["ja"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
                   "hits": [
                     {
-                      "document_en": {
-                        "name": "Attack on Titan",
-                        "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                        "author": "Hajime Isayama"
-                      },
-                      "document_ja": {
-                        "name": "進撃の巨人",
-                        "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                        "author": "諫山 創"
-                      },
-                      "document_zh": {
-                        "name": "进击的巨人",
-                        "description": "进击的巨人是日本的漫画系列，由諫山 創作画。",
-                        "author": "諫山創"
-                      },
-                      "id": 852,
-                      "_vectors": {
-                        "manual": [
-                          1.0,
-                          2.0,
-                          3.0
-                        ]
-                      },
-                      "_formatted": {
-                        "document_en": {
-                          "name": "Attack on Titan",
-                          "description": "Attack on Titan is a Japanese manga series written and illustrated by Hajime Isayama",
-                          "author": "Hajime Isayama"
-                        },
-                        "document_ja": {
-                          "name": "進撃の巨人",
-                          "description": "進撃の巨人は、日本の漫画シリーズであり、諫山 創によって作画されている。",
-                          "author": "諫山 創"
-                        },
-                        "document_zh": {
-                          "name": "<em>进</em><em>击</em><em>的</em><em>巨人</em>",
-                          "description": "<em>进</em><em>击</em><em>的</em><em>巨人</em>是日本的漫画系列，由諫山 創作画。",
-                          "author": "諫山創"
-                        },
-                        "id": "852",
-                        "_vectors": {
-                          "manual": [
-                            "1.0",
-                            "2.0",
-                            "3.0"
-                          ]
-                        }
-                      }
+                      "id": 852
                     }
                   ],
                   "query": "\"进击的巨人\"",
@@ -1236,7 +803,7 @@ async fn settings_change() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1256,7 +823,7 @@ async fn settings_change() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1299,7 +866,7 @@ async fn settings_change() {
     // chinese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["cmn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1319,7 +886,7 @@ async fn settings_change() {
     // force japanese
     index
         .search(
-            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToHighlight": ["*"]}),
+            json!({"q": "\"进击的巨人\"", "locales": ["jpn"], "attributesToRetrieve": ["id"]}),
             |response, code| {
                 snapshot!(response, @r###"
                 {
@@ -1355,7 +922,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Unknown value `invalid` at `.locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -1368,7 +935,7 @@ async fn invalid_locales() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Invalid value in parameter `locales`: Unsupported locale `invalid`, expected one of af, ak, am, ar, az, be, bn, bg, ca, cs, zh, da, de, el, en, eo, et, fi, fr, gu, he, hi, hr, hu, hy, id, it, jv, ja, kn, ka, km, ko, la, lv, lt, ml, mr, mk, my, ne, nl, nb, or, pa, fa, pl, pt, ro, ru, si, sk, sl, sn, es, sr, sv, ta, te, tl, th, tk, tr, uk, ur, uz, vi, yi, zu, afr, aka, amh, ara, aze, bel, ben, bul, cat, ces, cmn, dan, deu, ell, eng, epo, est, fin, fra, guj, heb, hin, hrv, hun, hye, ind, ita, jav, jpn, kan, kat, khm, kor, lat, lav, lit, mal, mar, mkd, mya, nep, nld, nob, ori, pan, pes, pol, por, ron, rus, sin, slk, slv, sna, spa, srp, swe, tam, tel, tgl, tha, tuk, tur, ukr, urd, uzb, vie, yid, zul",
       "code": "invalid_search_locales",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_search_locales"
@@ -1390,7 +957,7 @@ async fn invalid_localized_attributes_rules() {
         .await;
     snapshot!(response, @r###"
     {
-      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `epo`, `eng`, `rus`, `cmn`, `spa`, `por`, `ita`, `ben`, `fra`, `deu`, `ukr`, `kat`, `ara`, `hin`, `jpn`, `heb`, `yid`, `pol`, `amh`, `jav`, `kor`, `nob`, `dan`, `swe`, `fin`, `tur`, `nld`, `hun`, `ces`, `ell`, `bul`, `bel`, `mar`, `kan`, `ron`, `slv`, `hrv`, `srp`, `mkd`, `lit`, `lav`, `est`, `tam`, `vie`, `urd`, `tha`, `guj`, `uzb`, `pan`, `aze`, `ind`, `tel`, `pes`, `mal`, `ori`, `mya`, `nep`, `sin`, `khm`, `tuk`, `aka`, `zul`, `sna`, `afr`, `lat`, `slk`, `cat`, `tgl`, `hye`",
+      "message": "Unknown value `japan` at `.localizedAttributes[0].locales[0]`: expected one of `af`, `ak`, `am`, `ar`, `az`, `be`, `bn`, `bg`, `ca`, `cs`, `zh`, `da`, `de`, `el`, `en`, `eo`, `et`, `fi`, `fr`, `gu`, `he`, `hi`, `hr`, `hu`, `hy`, `id`, `it`, `jv`, `ja`, `kn`, `ka`, `km`, `ko`, `la`, `lv`, `lt`, `ml`, `mr`, `mk`, `my`, `ne`, `nl`, `nb`, `or`, `pa`, `fa`, `pl`, `pt`, `ro`, `ru`, `si`, `sk`, `sl`, `sn`, `es`, `sr`, `sv`, `ta`, `te`, `tl`, `th`, `tk`, `tr`, `uk`, `ur`, `uz`, `vi`, `yi`, `zu`, `afr`, `aka`, `amh`, `ara`, `aze`, `bel`, `ben`, `bul`, `cat`, `ces`, `cmn`, `dan`, `deu`, `ell`, `eng`, `epo`, `est`, `fin`, `fra`, `guj`, `heb`, `hin`, `hrv`, `hun`, `hye`, `ind`, `ita`, `jav`, `jpn`, `kan`, `kat`, `khm`, `kor`, `lat`, `lav`, `lit`, `mal`, `mar`, `mkd`, `mya`, `nep`, `nld`, `nob`, `ori`, `pan`, `pes`, `pol`, `por`, `ron`, `rus`, `sin`, `slk`, `slv`, `sna`, `spa`, `srp`, `swe`, `tam`, `tel`, `tgl`, `tha`, `tuk`, `tur`, `ukr`, `urd`, `uzb`, `vie`, `yid`, `zul`",
       "code": "invalid_settings_localized_attributes",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_settings_localized_attributes"

meilisearch/tests/search/mod.rs

@@ -16,208 +16,17 @@ mod restrict_searchable;
 mod search_queue;
 
 use meilisearch::Opt;
-use once_cell::sync::Lazy;
 use tempfile::TempDir;
 
-use crate::common::{default_settings, Server, Value};
+use crate::common::{
+    default_settings, shared_index_with_documents, shared_index_with_nested_documents, Server,
+    DOCUMENTS, FRUITS_DOCUMENTS, NESTED_DOCUMENTS, SCORE_DOCUMENTS, VECTOR_DOCUMENTS,
+};
 use crate::json;
 
-static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Shazam!",
-            "id": "287947",
-            "_vectors": { "manual": [1, 2, 3]},
-        },
-        {
-            "title": "Captain Marvel",
-            "id": "299537",
-            "_vectors": { "manual": [1, 2, 54] },
-        },
-        {
-            "title": "Escape Room",
-            "id": "522681",
-            "_vectors": { "manual": [10, -23, 32] },
-        },
-        {
-            "title": "How to Train Your Dragon: The Hidden World",
-            "id": "166428",
-            "_vectors": { "manual": [-100, 231, 32] },
-        },
-        {
-            "title": "Gläss",
-            "id": "450465",
-            "_vectors": { "manual": [-100, 340, 90] },
-        }
-    ])
-});
-
-static SCORE_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "title": "Batman the dark knight returns: Part 1",
-            "id": "A",
-        },
-        {
-            "title": "Batman the dark knight returns: Part 2",
-            "id": "B",
-        },
-        {
-            "title": "Batman Returns",
-            "id": "C",
-        },
-        {
-            "title": "Batman",
-            "id": "D",
-        },
-        {
-            "title": "Badman",
-            "id": "E",
-        }
-    ])
-});
-
-static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "id": 852,
-            "father": "jean",
-            "mother": "michelle",
-            "doggos": [
-                {
-                    "name": "bobby",
-                    "age": 2,
-                },
-                {
-                    "name": "buddy",
-                    "age": 4,
-                },
-            ],
-            "cattos": "pésti",
-            "_vectors": { "manual": [1, 2, 3]},
-        },
-        {
-            "id": 654,
-            "father": "pierre",
-            "mother": "sabine",
-            "doggos": [
-                {
-                    "name": "gros bill",
-                    "age": 8,
-                },
-            ],
-            "cattos": ["simba", "pestiféré"],
-            "_vectors": { "manual": [1, 2, 54] },
-        },
-        {
-            "id": 750,
-            "father": "romain",
-            "mother": "michelle",
-            "cattos": ["enigma"],
-            "_vectors": { "manual": [10, 23, 32] },
-        },
-        {
-            "id": 951,
-            "father": "jean-baptiste",
-            "mother": "sophie",
-            "doggos": [
-                {
-                    "name": "turbo",
-                    "age": 5,
-                },
-                {
-                    "name": "fast",
-                    "age": 6,
-                },
-            ],
-            "cattos": ["moumoute", "gomez"],
-            "_vectors": { "manual": [10, 23, 32] },
-        },
-    ])
-});
-
-static FRUITS_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-        {
-            "name": "Exclusive sale: green apple",
-            "id": "green-apple-boosted",
-            "BOOST": true
-        },
-        {
-            "name": "Pear",
-            "id": "pear",
-        },
-        {
-            "name": "Red apple gala",
-            "id": "red-apple-gala",
-        },
-        {
-            "name": "Exclusive sale: Red Tomato",
-            "id": "red-tomatoes-boosted",
-            "BOOST": true
-        },
-        {
-            "name": "Exclusive sale: Red delicious apple",
-            "id": "red-delicious-boosted",
-            "BOOST": true,
-        }
-    ])
-});
-
-static VECTOR_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-      {
-        "id": "A",
-        "description": "the dog barks at the cat",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.9, 0.8, 0.05],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-0.1, 0.55]
-        }
-      },
-      {
-        "id": "B",
-        "description": "the kitten scratched the beagle",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.8, 0.9, 0.5],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-0.2, 0.65]
-        }
-      },
-      {
-        "id": "C",
-        "description": "the dog had to stay alone today",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.85, 0.02, 0.1],
-          // dimensions [negative/positive, energy]
-          "sentiment": [-1.0, 0.1]
-        }
-      },
-      {
-        "id": "D",
-        "description": "the little boy pets the puppy",
-        "_vectors": {
-          // dimensions [canine, feline, young]
-          "animal": [0.8, 0.09, 0.8],
-          // dimensions [negative/positive, energy]
-          "sentiment": [0.8, 0.3]
-        }
-      },
-    ])
-});
-
 #[actix_rt::test]
 async fn simple_placeholder_search() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -225,11 +34,7 @@ async fn simple_placeholder_search() {
         })
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -240,13 +45,7 @@ async fn simple_placeholder_search() {
 
 #[actix_rt::test]
 async fn simple_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "glass"}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -254,11 +53,7 @@ async fn simple_search() {
         })
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(json!({"q": "pésti"}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -290,13 +85,7 @@ async fn phrase_search_with_stop_word() {
 
 #[actix_rt::test]
 async fn negative_phrase_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "-\"train your dragon\"" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -312,13 +101,7 @@ async fn negative_phrase_search() {
 
 #[actix_rt::test]
 async fn negative_word_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "-escape" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -343,13 +126,7 @@ async fn negative_word_search() {
 
 #[actix_rt::test]
 async fn non_negative_search() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"q": "- escape" }), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -442,13 +219,7 @@ async fn test_thai_language() {
 
 #[actix_rt::test]
 async fn search_multiple_params() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -465,11 +236,7 @@ async fn search_multiple_params() {
         )
         .await;
 
-    let index = server.index("nested");
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -555,15 +322,7 @@ async fn search_with_filter_string_notation() {
 
 #[actix_rt::test]
 async fn search_with_filter_array_notation() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     let (response, code) = index
         .search_post(json!({
             "filter": ["title = Gläss"]
@@ -609,15 +368,7 @@ async fn search_with_contains_filter() {
 
 #[actix_rt::test]
 async fn search_with_sort_on_numbers() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -630,14 +381,7 @@ async fn search_with_sort_on_numbers() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"sortableAttributes": ["doggos.age"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -653,15 +397,7 @@ async fn search_with_sort_on_numbers() {
 
 #[actix_rt::test]
 async fn search_with_sort_on_strings() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -674,14 +410,7 @@ async fn search_with_sort_on_strings() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"sortableAttributes": ["doggos.name"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
-
+    let index = shared_index_with_nested_documents().await;
     index
         .search(
             json!({
@@ -697,15 +426,7 @@ async fn search_with_sort_on_strings() {
 
 #[actix_rt::test]
 async fn search_with_multiple_sort() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"sortableAttributes": ["id", "title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     let (response, code) = index
         .search_post(json!({
             "sort": ["id:asc", "title:desc"]
@@ -717,15 +438,7 @@ async fn search_with_multiple_sort() {
 
 #[actix_rt::test]
 async fn search_facet_distribution() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({"filterableAttributes": ["title"]})).await;
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(
             json!({
@@ -740,13 +453,7 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    let index = server.index("nested");
-
-    index.update_settings(json!({"filterableAttributes": ["father", "doggos.name"]})).await;
-
-    let documents = NESTED_DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(3).await;
+    let index = shared_index_with_nested_documents().await;
 
     // TODO: TAMO: fix the test
     index
@@ -773,9 +480,6 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    index.update_settings(json!({"filterableAttributes": ["doggos"]})).await;
-    index.wait_task(4).await;
-
     index
         .search(
             json!({
@@ -811,9 +515,6 @@ async fn search_facet_distribution() {
         )
         .await;
 
-    index.update_settings(json!({"filterableAttributes": ["doggos.name"]})).await;
-    index.wait_task(5).await;
-
     index
         .search(
             json!({
@@ -1043,6 +744,10 @@ async fn test_score_details() {
                   {
                     "title": "How to Train Your Dragon: The Hidden World",
                     "id": "166428",
+                    "color": [
+                      "green",
+                      "red"
+                    ],
                     "_vectors": {
                       "manual": [
                         -100.0,
@@ -1469,6 +1174,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Shazam!",
         "id": "287947",
+        "color": [
+          "green",
+          "blue"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1486,6 +1195,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Captain Marvel",
         "id": "299537",
+        "color": [
+          "yellow",
+          "blue"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1503,6 +1216,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Gläss",
         "id": "450465",
+        "color": [
+          "blue",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1520,6 +1237,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "How to Train Your Dragon: The Hidden World",
         "id": "166428",
+        "color": [
+          "green",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1537,6 +1258,10 @@ async fn experimental_feature_vector_store() {
       {
         "title": "Escape Room",
         "id": "522681",
+        "color": [
+          "yellow",
+          "red"
+        ],
         "_vectors": {
           "manual": {
             "embeddings": [
@@ -1803,6 +1528,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,
@@ -1824,6 +1553,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,
@@ -1845,6 +1578,10 @@ async fn simple_search_with_strange_synonyms() {
               {
                 "title": "How to Train Your Dragon: The Hidden World",
                 "id": "166428",
+                "color": [
+                  "green",
+                  "red"
+                ],
                 "_vectors": {
                   "manual": [
                     -100.0,

meilisearch/tests/search/pagination.rs

@@ -1,16 +1,10 @@
+use super::shared_index_with_documents;
 use crate::common::Server;
 use crate::json;
-use crate::search::DOCUMENTS;
 
 #[actix_rt::test]
 async fn default_search_should_return_estimated_total_hit() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -28,13 +22,7 @@ async fn default_search_should_return_estimated_total_hit() {
 
 #[actix_rt::test]
 async fn simple_search() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"page": 1}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -53,13 +41,7 @@ async fn simple_search() {
 
 #[actix_rt::test]
 async fn page_zero_should_not_return_any_result() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"page": 0}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -73,13 +55,7 @@ async fn page_zero_should_not_return_any_result() {
 
 #[actix_rt::test]
 async fn hits_per_page_1() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"hitsPerPage": 1}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -93,13 +69,7 @@ async fn hits_per_page_1() {
 
 #[actix_rt::test]
 async fn hits_per_page_0_should_not_return_any_result() {
-    let server = Server::new().await;
-    let index = server.index("basic");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
+    let index = shared_index_with_documents().await;
     index
         .search(json!({"hitsPerPage": 0}), |response, code| {
             assert_eq!(code, 200, "{}", response);
@@ -113,8 +83,8 @@ async fn hits_per_page_0_should_not_return_any_result() {
 
 #[actix_rt::test]
 async fn ensure_placeholder_search_hit_count_valid() {
-    let server = Server::new().await;
-    let index = server.index("basic");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let documents = json!([
         {
@@ -143,15 +113,15 @@ async fn ensure_placeholder_search_hit_count_valid() {
             "distinct": 3,
         }
     ]);
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.add_documents(documents, None).await;
+    index.wait_task(task.uid()).await.succeeded();
 
-    let (_response, _code) = index
+    let (response, _code) = index
         .update_settings(
             json!({ "rankingRules": ["distinct:asc"], "distinctAttribute": "distinct"}),
         )
         .await;
-    index.wait_task(1).await;
+    index.wait_task(response.uid()).await.succeeded();
 
     for page in 0..=4 {
         index

meilisearch/tests/search/restrict_searchable.rs

@@ -8,8 +8,8 @@ use crate::json;
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");
 
-    index.add_documents(documents.clone(), None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.add_documents(documents.clone(), None).await;
+    index.wait_task(task.uid()).await.succeeded();
     index
 }
 

meilisearch/tests/search/search_queue.rs

@@ -37,6 +37,43 @@ async fn search_queue_register() {
         .unwrap();
 }
 
+#[actix_rt::test]
+async fn search_queue_register_with_explicit_drop() {
+    let queue = SearchQueue::new(4, NonZeroUsize::new(2).unwrap());
+
+    // First, use all the cores
+    let permit1 = queue.try_get_search_permit().await.unwrap();
+    let _permit2 = queue.try_get_search_permit().await.unwrap();
+
+    // If we free one spot we should be able to register one new search
+    permit1.drop().await;
+
+    let permit3 = queue.try_get_search_permit().await.unwrap();
+
+    // And again
+    permit3.drop().await;
+
+    let _permit4 = queue.try_get_search_permit().await.unwrap();
+}
+
+#[actix_rt::test]
+async fn search_queue_register_with_time_to_abort() {
+    let queue = Arc::new(
+        SearchQueue::new(1, NonZeroUsize::new(1).unwrap())
+            .with_time_to_abort(Duration::from_secs(1)),
+    );
+
+    // First, use all the cores
+    let permit1 = queue.try_get_search_permit().await.unwrap();
+    let q = queue.clone();
+    let permit2 = tokio::task::spawn(async move { q.try_get_search_permit().await });
+    tokio::time::sleep(Duration::from_secs(1)).await;
+    permit1.drop().await;
+    let ret = permit2.await.unwrap();
+
+    snapshot!(ret.unwrap_err(), @"Too many search requests running at the same time: 1. Retry after 10s.");
+}
+
 #[actix_rt::test]
 async fn wait_till_cores_are_available() {
     let queue = Arc::new(SearchQueue::new(4, NonZeroUsize::new(1).unwrap()));

meilisearch/tests/settings/errors.rs

@@ -5,8 +5,8 @@ use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "displayedAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -33,8 +33,8 @@ async fn settings_bad_displayed_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_searchable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "searchableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -61,8 +61,8 @@ async fn settings_bad_searchable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_filterable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "filterableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -89,8 +89,8 @@ async fn settings_bad_filterable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_sortable_attributes() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "sortableAttributes": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -117,8 +117,8 @@ async fn settings_bad_sortable_attributes() {
 
 #[actix_rt::test]
 async fn settings_bad_ranking_rules() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "rankingRules": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -145,8 +145,8 @@ async fn settings_bad_ranking_rules() {
 
 #[actix_rt::test]
 async fn settings_bad_stop_words() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "stopWords": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -173,8 +173,8 @@ async fn settings_bad_stop_words() {
 
 #[actix_rt::test]
 async fn settings_bad_synonyms() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "synonyms": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -201,8 +201,8 @@ async fn settings_bad_synonyms() {
 
 #[actix_rt::test]
 async fn settings_bad_distinct_attribute() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "distinctAttribute": ["doggo"] })).await;
     snapshot!(code, @"400 Bad Request");
@@ -229,8 +229,8 @@ async fn settings_bad_distinct_attribute() {
 
 #[actix_rt::test]
 async fn settings_bad_typo_tolerance() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "typoTolerance": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -284,8 +284,8 @@ async fn settings_bad_typo_tolerance() {
 
 #[actix_rt::test]
 async fn settings_bad_faceting() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "faceting": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -312,8 +312,8 @@ async fn settings_bad_faceting() {
 
 #[actix_rt::test]
 async fn settings_bad_pagination() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "pagination": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
@@ -340,8 +340,8 @@ async fn settings_bad_pagination() {
 
 #[actix_rt::test]
 async fn settings_bad_search_cutoff_ms() {
-    let server = Server::new().await;
-    let index = server.index("test");
+    let server = Server::new_shared();
+    let index = server.unique_index();
 
     let (response, code) = index.update_settings(json!({ "searchCutoffMs": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/settings/get_settings.rs

@@ -190,7 +190,8 @@ async fn secrets_are_hidden_in_settings() {
           "source": "rest",
           "apiKey": "My suXXXXXX...",
           "dimensions": 4,
-          "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+          "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+          "documentTemplateMaxBytes": 400,
           "url": "https://localhost:7777",
           "request": "{{text}}",
           "response": "{{embedding}}",

meilisearch/tests/similar/errors.rs

@@ -364,7 +364,7 @@ async fn filter_invalid_syntax_object() {
         .similar(json!({"id": 287947, "filter": "title & Glass"}), |response, code| {
             snapshot!(response, @r###"
             {
-              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
+              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
               "code": "invalid_similar_filter",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_similar_filter"
@@ -403,7 +403,7 @@ async fn filter_invalid_syntax_array() {
         .similar(json!({"id": 287947, "filter": ["title & Glass"]}), |response, code| {
             snapshot!(response, @r###"
             {
-              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
+              "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `title & Glass`.\n1:14 title & Glass",
               "code": "invalid_similar_filter",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_similar_filter"

meilisearch/tests/swap_indexes/errors.rs

@@ -5,7 +5,7 @@ use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.index_swap(json!("doggo")).await;
     snapshot!(code, @"400 Bad Request");
@@ -32,7 +32,7 @@ async fn swap_indexes_bad_format() {
 
 #[actix_rt::test]
 async fn swap_indexes_bad_indexes() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.index_swap(json!([{ "indexes": "doggo"}])).await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/tasks/errors.rs

@@ -4,7 +4,7 @@ use crate::common::Server;
 
 #[actix_rt::test]
 async fn task_bad_uids() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("uids=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -53,7 +53,7 @@ async fn task_bad_uids() {
 
 #[actix_rt::test]
 async fn task_bad_canceled_by() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("canceledBy=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -91,7 +91,7 @@ async fn task_bad_canceled_by() {
 
 #[actix_rt::test]
 async fn task_bad_types() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("types=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -129,7 +129,7 @@ async fn task_bad_types() {
 
 #[actix_rt::test]
 async fn task_bad_statuses() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("statuses=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -167,7 +167,7 @@ async fn task_bad_statuses() {
 
 #[actix_rt::test]
 async fn task_bad_index_uids() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("indexUids=the%20good%20doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -205,7 +205,7 @@ async fn task_bad_index_uids() {
 
 #[actix_rt::test]
 async fn task_bad_limit() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("limit=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -243,7 +243,7 @@ async fn task_bad_limit() {
 
 #[actix_rt::test]
 async fn task_bad_from() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("from=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -281,7 +281,7 @@ async fn task_bad_from() {
 
 #[actix_rt::test]
 async fn task_bad_after_enqueued_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterEnqueuedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -319,7 +319,7 @@ async fn task_bad_after_enqueued_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_enqueued_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeEnqueuedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -357,7 +357,7 @@ async fn task_bad_before_enqueued_at() {
 
 #[actix_rt::test]
 async fn task_bad_after_started_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterStartedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -395,7 +395,7 @@ async fn task_bad_after_started_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_started_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeStartedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -433,7 +433,7 @@ async fn task_bad_before_started_at() {
 
 #[actix_rt::test]
 async fn task_bad_after_finished_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("afterFinishedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");
@@ -471,7 +471,7 @@ async fn task_bad_after_finished_at() {
 
 #[actix_rt::test]
 async fn task_bad_before_finished_at() {
-    let server = Server::new().await;
+    let server = Server::new_shared();
 
     let (response, code) = server.tasks_filter("beforeFinishedAt=doggo").await;
     snapshot!(code, @"400 Bad Request");

meilisearch/tests/tasks/mod.rs

@@ -66,8 +66,8 @@ async fn list_tasks() {
 async fn list_tasks_with_star_filters() {
     let server = Server::new().await;
     let index = server.index("test");
-    index.create(None).await;
-    index.wait_task(0).await;
+    let (task, _code) = index.create(None).await;
+    index.wait_task(task.uid()).await;
     index
         .add_documents(serde_json::from_str(include_str!("../assets/test_set.json")).unwrap(), None)
         .await;

meilisearch/tests/vector/mod.rs

@@ -2,10 +2,13 @@ mod openai;
 mod rest;
 mod settings;
 
+use std::str::FromStr;
+
 use meili_snap::{json_string, snapshot};
+use meilisearch::option::MaxThreads;
 
 use crate::common::index::Index;
-use crate::common::{GetAllDocumentsOptions, Server};
+use crate::common::{default_settings, GetAllDocumentsOptions, Server};
 use crate::json;
 
 async fn get_server_vector() -> Server {
@@ -503,7 +506,11 @@ async fn user_provided_embeddings_error() {
 
 #[actix_rt::test]
 async fn user_provided_vectors_error() {
-    let server = Server::new().await;
+    let temp = tempfile::tempdir().unwrap();
+    let mut options = default_settings(temp.path());
+    // If we have more than one indexing thread the error messages below may become inconsistent
+    options.indexer_options.max_indexing_threads = MaxThreads::from_str("1").unwrap();
+    let server = Server::new_with_options(options).await.unwrap();
 
     let index = generate_default_user_provided_documents(&server).await;
 

meilisearch/tests/vector/openai.rs

@@ -302,7 +302,8 @@ async fn create_mock_with_template(
         "source": "openAi",
         "url": url,
         "apiKey": API_KEY,
-        "documentTemplate": document_template
+        "documentTemplate": document_template,
+        "documentTemplateMaxBytes": 8000000,
     });
 
     model_dimensions.add_to_settings(&mut embedder_settings);
@@ -693,6 +694,7 @@ async fn bad_api_key() {
             "model": "text-embedding-3-large",
             "apiKey": "XXX...",
             "documentTemplate": "{%- if doc.gender == \"F\" -%}Une chienne nommée {{doc.name}}, née en {{doc.birthyear}}\n        {%- else -%}\n        Un chien nommé {{doc.name}}, né en {{doc.birthyear}}\n        {%- endif %}, de race {{doc.breed}}.",
+            "documentTemplateMaxBytes": 8000000,
             "url": "[url]"
           }
         }
@@ -735,6 +737,7 @@ async fn bad_api_key() {
             "source": "openAi",
             "model": "text-embedding-3-large",
             "documentTemplate": "{%- if doc.gender == \"F\" -%}Une chienne nommée {{doc.name}}, née en {{doc.birthyear}}\n        {%- else -%}\n        Un chien nommé {{doc.name}}, né en {{doc.birthyear}}\n        {%- endif %}, de race {{doc.breed}}.",
+            "documentTemplateMaxBytes": 8000000,
             "url": "[url]"
           }
         }

meilisearch/tests/vector/rest.rs

@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use meili_snap::{json_string, snapshot};
@@ -37,6 +38,46 @@ async fn create_mock() -> (MockServer, Value) {
     (mock_server, embedder_settings)
 }
 
+async fn create_mock_map() -> (MockServer, Value) {
+    let mock_server = MockServer::start().await;
+
+    let text_to_embedding: BTreeMap<_, _> = vec![
+        // text -> embedding
+        ("name: kefir\n", [0.0, 0.1, 0.2]),
+    ]
+    // turn into btree
+    .into_iter()
+    .collect();
+
+    Mock::given(method("POST"))
+        .and(path("/"))
+        .respond_with(move |req: &Request| {
+            let text: String = req.body_json().unwrap();
+            match text_to_embedding.get(text.as_str()) {
+                Some(embedding) => {
+                    ResponseTemplate::new(200).set_body_json(json!({ "data": embedding }))
+                }
+                None => ResponseTemplate::new(404)
+                    .set_body_json(json!({"error": "text not found", "text": text})),
+            }
+        })
+        .mount(&mock_server)
+        .await;
+    let url = mock_server.uri();
+
+    let embedder_settings = json!({
+        "source": "rest",
+        "url": url,
+        "dimensions": 3,
+        "request": "{{text}}",
+        "response": {
+          "data": "{{embedding}}"
+        }
+    });
+
+    (mock_server, embedder_settings)
+}
+
 #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
 struct MultipleRequest {
     input: Vec<String>,
@@ -1100,6 +1141,7 @@ async fn server_returns_bad_request() {
 
     let (response, code) = index
         .update_settings(json!({
+          "searchableAttributes": ["name", "missing_field"],
           "embedders": {
               "rest": json!({ "source": "rest", "url": mock.uri(), "request": "{{text}}", "response": "{{embedding}}", "dimensions": 3 }),
           },
@@ -1115,6 +1157,10 @@ async fn server_returns_bad_request() {
       "type": "settingsUpdate",
       "canceledBy": null,
       "details": {
+        "searchableAttributes": [
+          "name",
+          "missing_field"
+        ],
         "embedders": {
           "rest": {
             "source": "rest",
@@ -1148,7 +1194,7 @@ async fn server_returns_bad_request() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "While embedding documents for embedder `rest`: user error: sent a bad request to embedding server\n  - Hint: check that the `request` in the embedder configuration matches the remote server's API\n  - server replied with `{\"error\":\"Invalid request: invalid type: string \\\" id: 1\\\\n name: kefir\\\\n\\\", expected struct MultipleRequest at line 1 column 24\"}`",
+        "message": "While embedding documents for embedder `rest`: user error: sent a bad request to embedding server\n  - Hint: check that the `request` in the embedder configuration matches the remote server's API\n  - server replied with `{\"error\":\"Invalid request: invalid type: string \\\"name: kefir\\\\n\\\", expected struct MultipleRequest at line 1 column 15\"}`",
         "code": "vector_embedding_error",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
@@ -1891,3 +1937,109 @@ async fn server_custom_header() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn searchable_reindex() {
+    let (_mock, setting) = create_mock_map().await;
+    let server = get_server_vector().await;
+    let index = server.index("doggo");
+
+    let (response, code) = index
+        .update_settings(json!({
+          "searchableAttributes": ["name", "missing_field"],
+          "embedders": {
+              "rest": setting,
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "succeeded",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "searchableAttributes": [
+          "name",
+          "missing_field"
+        ],
+        "embedders": {
+          "rest": {
+            "source": "rest",
+            "dimensions": 3,
+            "url": "[url]",
+            "request": "{{text}}",
+            "response": {
+              "data": "{{embedding}}"
+            }
+          }
+        }
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    let (response, code) =
+        index.add_documents(json!( { "id": 1, "name": "kefir", "breed": "patou" }), None).await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "succeeded",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 1
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    // triggers reindexing with the new searchable attribute.
+    // as the mock intentionally doesn't know of this text, the task will fail, outputting the putative rendered text.
+    let (response, code) = index
+        .update_settings(json!({
+          "searchableAttributes": ["breed"],
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let task = server.wait_task(response.uid()).await;
+    snapshot!(task, @r###"
+    {
+      "uid": "[uid]",
+      "indexUid": "doggo",
+      "status": "failed",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "searchableAttributes": [
+          "breed"
+        ]
+      },
+      "error": {
+        "message": "While embedding documents for embedder `rest`: error: received unexpected HTTP 404 from embedding server\n  - server replied with `{\"error\":\"text not found\",\"text\":\"breed: patou\\n\"}`",
+        "code": "vector_embedding_error",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+}

meilitool/Cargo.toml

@@ -15,5 +15,6 @@ dump = { path = "../dump" }
 file-store = { path = "../file-store" }
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
+serde = { version = "1.0.209", features = ["derive"] }
 time = { version = "0.3.36", features = ["formatting"] }
 uuid = { version = "1.10.0", features = ["v4"], default-features = false }

meilitool/src/main.rs

@@ -2,7 +2,7 @@ use std::fs::{read_dir, read_to_string, remove_file, File};
 use std::io::BufWriter;
 use std::path::PathBuf;
 
-use anyhow::Context;
+use anyhow::{bail, Context};
 use clap::{Parser, Subcommand};
 use dump::{DumpWriter, IndexMetadata};
 use file_store::FileStore;
@@ -10,9 +10,10 @@ use meilisearch_auth::AuthController;
 use meilisearch_types::heed::types::{SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
+use meilisearch_types::milli::index::{db_name, main_key};
 use meilisearch_types::milli::{obkv_to_json, BEU32};
 use meilisearch_types::tasks::{Status, Task};
-use meilisearch_types::versioning::check_version_file;
+use meilisearch_types::versioning::{create_version_file, get_version, parse_version};
 use meilisearch_types::Index;
 use time::macros::format_description;
 use time::OffsetDateTime;
@@ -62,21 +63,458 @@ enum Command {
         #[arg(long)]
         skip_enqueued_tasks: bool,
     },
+
+    /// Attempts to upgrade from one major version to the next without a dump.
+    ///
+    /// Make sure to run this commmand when Meilisearch is not running!
+    /// If Meilisearch is running while executing this command, the database could be corrupted
+    /// (contain data from both the old and the new versions)
+    ///
+    /// Supported upgrade paths:
+    ///
+    /// - v1.9.0 -> v1.10.0
+    OfflineUpgrade {
+        #[arg(long)]
+        target_version: String,
+    },
 }
 
 fn main() -> anyhow::Result<()> {
     let Cli { db_path, command } = Cli::parse();
 
-    check_version_file(&db_path).context("While checking the version file")?;
+    let detected_version = get_version(&db_path).context("While checking the version file")?;
 
     match command {
         Command::ClearTaskQueue => clear_task_queue(db_path),
         Command::ExportADump { dump_dir, skip_enqueued_tasks } => {
             export_a_dump(db_path, dump_dir, skip_enqueued_tasks)
         }
+        Command::OfflineUpgrade { target_version } => {
+            let target_version = parse_version(&target_version).context("While parsing `--target-version`. Make sure `--target-version` is in the format MAJOR.MINOR.PATCH")?;
+            OfflineUpgrade { db_path, current_version: detected_version, target_version }.upgrade()
+        }
     }
 }
 
+struct OfflineUpgrade {
+    db_path: PathBuf,
+    current_version: (String, String, String),
+    target_version: (String, String, String),
+}
+
+impl OfflineUpgrade {
+    fn upgrade(self) -> anyhow::Result<()> {
+        // TODO: if we make this process support more versions, introduce a more flexible way of checking for the version
+        // currently only supports v1.9 to v1.10
+        let (current_major, current_minor, current_patch) = &self.current_version;
+
+        match (current_major.as_str(), current_minor.as_str(), current_patch.as_str()) {
+            ("1", "9", _) => {}
+            _ => {
+                bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9")
+            }
+        }
+
+        let (target_major, target_minor, target_patch) = &self.target_version;
+
+        match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) {
+            ("1", "10", _) => {}
+            _ => {
+                bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.10")
+            }
+        }
+
+        println!("Upgrading from {current_major}.{current_minor}.{current_patch} to {target_major}.{target_minor}.{target_patch}");
+
+        self.v1_9_to_v1_10()?;
+
+        println!("Writing VERSION file");
+
+        create_version_file(&self.db_path, target_major, target_minor, target_patch)
+            .context("while writing VERSION file after the upgrade")?;
+
+        println!("Success");
+
+        Ok(())
+    }
+
+    fn v1_9_to_v1_10(&self) -> anyhow::Result<()> {
+        // 2 changes here
+
+        // 1. date format. needs to be done before opening the Index
+        // 2. REST embedders. We don't support this case right now, so bail
+
+        let index_scheduler_path = self.db_path.join("tasks");
+        let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) }
+            .with_context(|| {
+                format!("While trying to open {:?}", index_scheduler_path.display())
+            })?;
+
+        let mut sched_wtxn = env.write_txn()?;
+
+        let index_mapping: Database<Str, UuidCodec> =
+            try_opening_database(&env, &sched_wtxn, "index-mapping")?;
+
+        let index_stats: Database<UuidCodec, Unspecified> =
+            try_opening_database(&env, &sched_wtxn, "index-stats").with_context(|| {
+                format!("While trying to open {:?}", index_scheduler_path.display())
+            })?;
+
+        let index_count =
+            index_mapping.len(&sched_wtxn).context("while reading the number of indexes")?;
+
+        // FIXME: not ideal, we have to pre-populate all indexes to prevent double borrow of sched_wtxn
+        // 1. immutably for the iteration
+        // 2. mutably for updating index stats
+        let indexes: Vec<_> = index_mapping
+            .iter(&sched_wtxn)?
+            .map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid)))
+            .collect();
+
+        let mut rest_embedders = Vec::new();
+
+        let mut unwrapped_indexes = Vec::new();
+
+        // check that update can take place
+        for (index_index, result) in indexes.into_iter().enumerate() {
+            let (uid, uuid) = result?;
+            let index_path = self.db_path.join("indexes").join(uuid.to_string());
+
+            println!(
+                "[{}/{index_count}]Checking that update can take place for  `{uid}` at `{}`",
+                index_index + 1,
+                index_path.display()
+            );
+
+            let index_env = unsafe {
+                // FIXME: fetch the 25 magic number from the index file
+                EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
+                    format!("while opening index {uid} at '{}'", index_path.display())
+                })?
+            };
+
+            let index_txn = index_env.read_txn().with_context(|| {
+                format!(
+                    "while obtaining a write transaction for index {uid} at {}",
+                    index_path.display()
+                )
+            })?;
+
+            println!("\t- Checking for incompatible embedders (REST embedders)");
+            let rest_embedders_for_index = find_rest_embedders(&uid, &index_env, &index_txn)?;
+
+            if rest_embedders_for_index.is_empty() {
+                unwrapped_indexes.push((uid, uuid));
+            } else {
+                // no need to add to unwrapped indexes because we'll exit early
+                rest_embedders.push((uid, rest_embedders_for_index));
+            }
+        }
+
+        if !rest_embedders.is_empty() {
+            let rest_embedders = rest_embedders
+                .into_iter()
+                .flat_map(|(index, embedders)| std::iter::repeat(index.clone()).zip(embedders))
+                .map(|(index, embedder)| format!("\t- embedder `{embedder}` in index `{index}`"))
+                .collect::<Vec<_>>()
+                .join("\n");
+            bail!("The update cannot take place because there are REST embedder(s). Remove them before proceeding with the update:\n{rest_embedders}\n\n\
+            The database has not been modified and is still a valid v1.9 database.");
+        }
+
+        println!("Update can take place, updating");
+
+        for (index_index, (uid, uuid)) in unwrapped_indexes.into_iter().enumerate() {
+            let index_path = self.db_path.join("indexes").join(uuid.to_string());
+
+            println!(
+                "[{}/{index_count}]Updating index `{uid}` at `{}`",
+                index_index + 1,
+                index_path.display()
+            );
+
+            let index_env = unsafe {
+                // FIXME: fetch the 25 magic number from the index file
+                EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
+                    format!("while opening index {uid} at '{}'", index_path.display())
+                })?
+            };
+
+            let mut index_wtxn = index_env.write_txn().with_context(|| {
+                format!(
+                    "while obtaining a write transaction for index `{uid}` at `{}`",
+                    index_path.display()
+                )
+            })?;
+
+            println!("\t- Updating index stats");
+            update_index_stats(index_stats, &uid, uuid, &mut sched_wtxn)?;
+            println!("\t- Updating date format");
+            update_date_format(&uid, &index_env, &mut index_wtxn)?;
+
+            index_wtxn.commit().with_context(|| {
+                format!(
+                    "while committing the write txn for index `{uid}` at {}",
+                    index_path.display()
+                )
+            })?;
+        }
+
+        sched_wtxn.commit().context("while committing the write txn for the index-scheduler")?;
+
+        println!("Upgrading database succeeded");
+
+        Ok(())
+    }
+}
+
+pub mod v1_9 {
+    pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
+
+    /// The statistics that can be computed from an `Index` object.
+    #[derive(serde::Serialize, serde::Deserialize, Debug)]
+    pub struct IndexStats {
+        /// Number of documents in the index.
+        pub number_of_documents: u64,
+        /// Size taken up by the index' DB, in bytes.
+        ///
+        /// This includes the size taken by both the used and free pages of the DB, and as the free pages
+        /// are not returned to the disk after a deletion, this number is typically larger than
+        /// `used_database_size` that only includes the size of the used pages.
+        pub database_size: u64,
+        /// Size taken by the used pages of the index' DB, in bytes.
+        ///
+        /// As the DB backend does not return to the disk the pages that are not currently used by the DB,
+        /// this value is typically smaller than `database_size`.
+        pub used_database_size: u64,
+        /// Association of every field name with the number of times it occurs in the documents.
+        pub field_distribution: FieldDistribution,
+        /// Creation date of the index.
+        pub created_at: time::OffsetDateTime,
+        /// Date of the last update of the index.
+        pub updated_at: time::OffsetDateTime,
+    }
+
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Debug, Deserialize, Serialize)]
+    pub struct IndexEmbeddingConfig {
+        pub name: String,
+        pub config: EmbeddingConfig,
+    }
+
+    #[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
+    pub struct EmbeddingConfig {
+        /// Options of the embedder, specific to each kind of embedder
+        pub embedder_options: EmbedderOptions,
+    }
+
+    /// Options of an embedder, specific to each kind of embedder.
+    #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+    pub enum EmbedderOptions {
+        HuggingFace(hf::EmbedderOptions),
+        OpenAi(openai::EmbedderOptions),
+        Ollama(ollama::EmbedderOptions),
+        UserProvided(manual::EmbedderOptions),
+        Rest(rest::EmbedderOptions),
+    }
+
+    impl Default for EmbedderOptions {
+        fn default() -> Self {
+            Self::OpenAi(openai::EmbedderOptions { api_key: None, dimensions: None })
+        }
+    }
+
+    mod hf {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub model: String,
+            pub revision: Option<String>,
+        }
+    }
+    mod openai {
+
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub api_key: Option<String>,
+            pub dimensions: Option<usize>,
+        }
+    }
+    mod ollama {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub embedding_model: String,
+            pub url: Option<String>,
+            pub api_key: Option<String>,
+        }
+    }
+    mod manual {
+        #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+        pub struct EmbedderOptions {
+            pub dimensions: usize,
+        }
+    }
+    mod rest {
+        #[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize, Hash)]
+        pub struct EmbedderOptions {
+            pub api_key: Option<String>,
+            pub dimensions: Option<usize>,
+            pub url: String,
+            pub input_field: Vec<String>,
+            // path to the array of embeddings
+            pub path_to_embeddings: Vec<String>,
+            // shape of a single embedding
+            pub embedding_object: Vec<String>,
+        }
+    }
+
+    pub type OffsetDateTime = time::OffsetDateTime;
+}
+
+pub mod v1_10 {
+    use crate::v1_9;
+
+    pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
+
+    /// The statistics that can be computed from an `Index` object.
+    #[derive(serde::Serialize, serde::Deserialize, Debug)]
+    pub struct IndexStats {
+        /// Number of documents in the index.
+        pub number_of_documents: u64,
+        /// Size taken up by the index' DB, in bytes.
+        ///
+        /// This includes the size taken by both the used and free pages of the DB, and as the free pages
+        /// are not returned to the disk after a deletion, this number is typically larger than
+        /// `used_database_size` that only includes the size of the used pages.
+        pub database_size: u64,
+        /// Size taken by the used pages of the index' DB, in bytes.
+        ///
+        /// As the DB backend does not return to the disk the pages that are not currently used by the DB,
+        /// this value is typically smaller than `database_size`.
+        pub used_database_size: u64,
+        /// Association of every field name with the number of times it occurs in the documents.
+        pub field_distribution: FieldDistribution,
+        /// Creation date of the index.
+        #[serde(with = "time::serde::rfc3339")]
+        pub created_at: time::OffsetDateTime,
+        /// Date of the last update of the index.
+        #[serde(with = "time::serde::rfc3339")]
+        pub updated_at: time::OffsetDateTime,
+    }
+
+    impl From<v1_9::IndexStats> for IndexStats {
+        fn from(
+            v1_9::IndexStats {
+                number_of_documents,
+                database_size,
+                used_database_size,
+                field_distribution,
+                created_at,
+                updated_at,
+            }: v1_9::IndexStats,
+        ) -> Self {
+            IndexStats {
+                number_of_documents,
+                database_size,
+                used_database_size,
+                field_distribution,
+                created_at,
+                updated_at,
+            }
+        }
+    }
+
+    #[derive(serde::Serialize, serde::Deserialize)]
+    #[serde(transparent)]
+    pub struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime);
+}
+
+fn update_index_stats(
+    index_stats: Database<UuidCodec, Unspecified>,
+    index_uid: &str,
+    index_uuid: uuid::Uuid,
+    sched_wtxn: &mut RwTxn,
+) -> anyhow::Result<()> {
+    let ctx = || format!("while updating index stats for index `{index_uid}`");
+
+    let stats: Option<v1_9::IndexStats> = index_stats
+        .remap_data_type::<SerdeJson<v1_9::IndexStats>>()
+        .get(sched_wtxn, &index_uuid)
+        .with_context(ctx)?;
+
+    if let Some(stats) = stats {
+        let stats: v1_10::IndexStats = stats.into();
+
+        index_stats
+            .remap_data_type::<SerdeJson<v1_10::IndexStats>>()
+            .put(sched_wtxn, &index_uuid, &stats)
+            .with_context(ctx)?;
+    }
+
+    Ok(())
+}
+
+fn update_date_format(
+    index_uid: &str,
+    index_env: &Env,
+    index_wtxn: &mut RwTxn,
+) -> anyhow::Result<()> {
+    let main = try_opening_poly_database(index_env, index_wtxn, db_name::MAIN)
+        .with_context(|| format!("while updating date format for index `{index_uid}`"))?;
+
+    date_round_trip(index_wtxn, index_uid, main, main_key::CREATED_AT_KEY)?;
+    date_round_trip(index_wtxn, index_uid, main, main_key::UPDATED_AT_KEY)?;
+
+    Ok(())
+}
+
+fn find_rest_embedders(
+    index_uid: &str,
+    index_env: &Env,
+    index_txn: &RoTxn,
+) -> anyhow::Result<Vec<String>> {
+    let main = try_opening_poly_database(index_env, index_txn, db_name::MAIN)
+        .with_context(|| format!("while checking REST embedders for index `{index_uid}`"))?;
+
+    let mut rest_embedders = vec![];
+
+    for config in main
+        .remap_types::<Str, SerdeJson<Vec<v1_9::IndexEmbeddingConfig>>>()
+        .get(index_txn, main_key::EMBEDDING_CONFIGS)?
+        .unwrap_or_default()
+    {
+        if let v1_9::EmbedderOptions::Rest(_) = config.config.embedder_options {
+            rest_embedders.push(config.name);
+        }
+    }
+
+    Ok(rest_embedders)
+}
+
+fn date_round_trip(
+    wtxn: &mut RwTxn,
+    index_uid: &str,
+    db: Database<Unspecified, Unspecified>,
+    key: &str,
+) -> anyhow::Result<()> {
+    let datetime =
+        db.remap_types::<Str, SerdeJson<v1_9::OffsetDateTime>>().get(wtxn, key).with_context(
+            || format!("could not read `{key}` while updating date format for index `{index_uid}`"),
+        )?;
+
+    if let Some(datetime) = datetime {
+        db.remap_types::<Str, SerdeJson<v1_10::OffsetDateTime>>()
+            .put(wtxn, key, &v1_10::OffsetDateTime(datetime))
+            .with_context(|| {
+                format!(
+                    "could not write `{key}` while updating date format for index `{index_uid}`"
+                )
+            })?;
+    }
+
+    Ok(())
+}
+
 /// Clears the task queue located at `db_path`.
 fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> {
     let path = db_path.join("tasks");

milli/src/error.rs

@@ -258,6 +258,8 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
     },
     #[error("`.embedders.{embedder_name}.dimensions`: `dimensions` cannot be zero")]
     InvalidSettingsDimensions { embedder_name: String },
+    #[error("`.embedders.{embedder_name}.documentTemplateMaxBytes`: `documentTemplateMaxBytes` cannot be zero")]
+    InvalidSettingsDocumentTemplateMaxBytes { embedder_name: String },
     #[error("`.embedders.{embedder_name}.url`: could not parse `{url}`: {inner_error}")]
     InvalidUrl { embedder_name: String, inner_error: url::ParseError, url: String },
     #[error("Document editions cannot modify a document's primary key")]

milli/src/prompt/context.rs

@@ -5,7 +5,7 @@ use liquid::{ObjectView, ValueView};
 
 use super::document::Document;
 use super::fields::Fields;
-use crate::FieldsIdsMap;
+use super::FieldsIdsMapWithMetadata;
 
 #[derive(Debug, Clone)]
 pub struct Context<'a> {
@@ -14,7 +14,7 @@ pub struct Context<'a> {
 }
 
 impl<'a> Context<'a> {
-    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMap) -> Self {
+    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMapWithMetadata<'a>) -> Self {
         Self { document, fields: Fields::new(document, field_id_map) }
     }
 }

milli/src/prompt/fields.rs

@@ -4,16 +4,20 @@ use liquid::model::{
 use liquid::{ObjectView, ValueView};
 
 use super::document::Document;
-use crate::FieldsIdsMap;
+use super::{FieldMetadata, FieldsIdsMapWithMetadata};
 #[derive(Debug, Clone)]
 pub struct Fields<'a>(Vec<FieldValue<'a>>);
 
 impl<'a> Fields<'a> {
-    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMap) -> Self {
+    pub fn new(document: &'a Document<'a>, field_id_map: &'a FieldsIdsMapWithMetadata<'a>) -> Self {
         Self(
             std::iter::repeat(document)
                 .zip(field_id_map.iter())
-                .map(|(document, (_fid, name))| FieldValue { document, name })
+                .map(|(document, (fid, name))| FieldValue {
+                    document,
+                    name,
+                    metadata: field_id_map.metadata(fid).unwrap_or_default(),
+                })
                 .collect(),
         )
     }
@@ -23,6 +27,7 @@ impl<'a> Fields<'a> {
 pub struct FieldValue<'a> {
     name: &'a str,
     document: &'a Document<'a>,
+    metadata: FieldMetadata,
 }
 
 impl<'a> ValueView for FieldValue<'a> {
@@ -74,6 +79,10 @@ impl<'a> FieldValue<'a> {
         self.document.get(self.name).unwrap_or(&LiquidValue::Nil)
     }
 
+    pub fn is_searchable(&self) -> &bool {
+        &self.metadata.searchable
+    }
+
     pub fn is_empty(&self) -> bool {
         self.size() == 0
     }
@@ -89,12 +98,14 @@ impl<'a> ObjectView for FieldValue<'a> {
     }
 
     fn keys<'k>(&'k self) -> Box<dyn Iterator<Item = KStringCow<'k>> + 'k> {
-        Box::new(["name", "value"].iter().map(|&x| KStringCow::from_static(x)))
+        Box::new(["name", "value", "is_searchable"].iter().map(|&x| KStringCow::from_static(x)))
     }
 
     fn values<'k>(&'k self) -> Box<dyn Iterator<Item = &'k dyn ValueView> + 'k> {
         Box::new(
-            std::iter::once(self.name() as &dyn ValueView).chain(std::iter::once(self.value())),
+            std::iter::once(self.name() as &dyn ValueView)
+                .chain(std::iter::once(self.value()))
+                .chain(std::iter::once(self.is_searchable() as &dyn ValueView)),
         )
     }
 
@@ -103,13 +114,14 @@ impl<'a> ObjectView for FieldValue<'a> {
     }
 
     fn contains_key(&self, index: &str) -> bool {
-        index == "name" || index == "value"
+        index == "name" || index == "value" || index == "is_searchable"
     }
 
     fn get<'s>(&'s self, index: &str) -> Option<&'s dyn ValueView> {
         match index {
             "name" => Some(self.name()),
             "value" => Some(self.value()),
+            "is_searchable" => Some(self.is_searchable()),
             _ => None,
         }
     }

milli/src/prompt/mod.rs

@@ -4,28 +4,33 @@ pub(crate) mod error;
 mod fields;
 mod template_checker;
 
+use std::collections::BTreeMap;
 use std::convert::TryFrom;
+use std::num::NonZeroUsize;
+use std::ops::Deref;
 
 use error::{NewPromptError, RenderPromptError};
 
 use self::context::Context;
 use self::document::Document;
 use crate::update::del_add::DelAdd;
-use crate::FieldsIdsMap;
+use crate::{FieldId, FieldsIdsMap};
 
 pub struct Prompt {
     template: liquid::Template,
     template_text: String,
+    max_bytes: Option<NonZeroUsize>,
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct PromptData {
     pub template: String,
+    pub max_bytes: Option<NonZeroUsize>,
 }
 
 impl From<Prompt> for PromptData {
     fn from(value: Prompt) -> Self {
-        Self { template: value.template_text }
+        Self { template: value.template_text, max_bytes: value.max_bytes }
     }
 }
 
@@ -33,14 +38,18 @@ impl TryFrom<PromptData> for Prompt {
     type Error = NewPromptError;
 
     fn try_from(value: PromptData) -> Result<Self, Self::Error> {
-        Prompt::new(value.template)
+        Prompt::new(value.template, value.max_bytes)
     }
 }
 
 impl Clone for Prompt {
     fn clone(&self) -> Self {
         let template_text = self.template_text.clone();
-        Self { template: new_template(&template_text).unwrap(), template_text }
+        Self {
+            template: new_template(&template_text).unwrap(),
+            template_text,
+            max_bytes: self.max_bytes,
+        }
     }
 }
 
@@ -53,25 +62,35 @@ fn default_template() -> liquid::Template {
 }
 
 fn default_template_text() -> &'static str {
-    "{% for field in fields %} \
+    "{% for field in fields %}\
+    {% if field.is_searchable and field.value != nil %}\
     {{ field.name }}: {{ field.value }}\n\
+    {% endif %}\
     {% endfor %}"
 }
 
+pub fn default_max_bytes() -> NonZeroUsize {
+    NonZeroUsize::new(400).unwrap()
+}
+
 impl Default for Prompt {
     fn default() -> Self {
-        Self { template: default_template(), template_text: default_template_text().into() }
+        Self {
+            template: default_template(),
+            template_text: default_template_text().into(),
+            max_bytes: Some(default_max_bytes()),
+        }
     }
 }
 
 impl Default for PromptData {
     fn default() -> Self {
-        Self { template: default_template_text().into() }
+        Self { template: default_template_text().into(), max_bytes: Some(default_max_bytes()) }
     }
 }
 
 impl Prompt {
-    pub fn new(template: String) -> Result<Self, NewPromptError> {
+    pub fn new(template: String, max_bytes: Option<NonZeroUsize>) -> Result<Self, NewPromptError> {
         let this = Self {
             template: liquid::ParserBuilder::with_stdlib()
                 .build()
@@ -79,6 +98,7 @@ impl Prompt {
                 .parse(&template)
                 .map_err(NewPromptError::cannot_parse_template)?,
             template_text: template,
+            max_bytes,
         };
 
         // render template with special object that's OK with `doc.*` and `fields.*`
@@ -93,20 +113,72 @@ impl Prompt {
         &self,
         document: obkv::KvReaderU16<'_>,
         side: DelAdd,
-        field_id_map: &FieldsIdsMap,
+        field_id_map: &FieldsIdsMapWithMetadata,
     ) -> Result<String, RenderPromptError> {
         let document = Document::new(document, side, field_id_map);
         let context = Context::new(&document, field_id_map);
 
-        self.template.render(&context).map_err(RenderPromptError::missing_context)
+        let mut rendered =
+            self.template.render(&context).map_err(RenderPromptError::missing_context)?;
+        if let Some(max_bytes) = self.max_bytes {
+            truncate(&mut rendered, max_bytes.get());
+        }
+        Ok(rendered)
     }
 }
 
+fn truncate(s: &mut String, max_bytes: usize) {
+    if max_bytes >= s.len() {
+        return;
+    }
+    for i in (0..=max_bytes).rev() {
+        if s.is_char_boundary(i) {
+            s.truncate(i);
+            break;
+        }
+    }
+}
+
+pub struct FieldsIdsMapWithMetadata<'a> {
+    fields_ids_map: &'a FieldsIdsMap,
+    metadata: BTreeMap<FieldId, FieldMetadata>,
+}
+
+impl<'a> FieldsIdsMapWithMetadata<'a> {
+    pub fn new(fields_ids_map: &'a FieldsIdsMap, searchable_fields_ids: &'_ [FieldId]) -> Self {
+        let mut metadata: BTreeMap<FieldId, FieldMetadata> =
+            fields_ids_map.ids().map(|id| (id, Default::default())).collect();
+        for searchable_field_id in searchable_fields_ids {
+            let Some(metadata) = metadata.get_mut(searchable_field_id) else { continue };
+            metadata.searchable = true;
+        }
+        Self { fields_ids_map, metadata }
+    }
+
+    pub fn metadata(&self, field_id: FieldId) -> Option<FieldMetadata> {
+        self.metadata.get(&field_id).copied()
+    }
+}
+
+impl<'a> Deref for FieldsIdsMapWithMetadata<'a> {
+    type Target = FieldsIdsMap;
+
+    fn deref(&self) -> &Self::Target {
+        self.fields_ids_map
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy)]
+pub struct FieldMetadata {
+    pub searchable: bool,
+}
+
 #[cfg(test)]
 mod test {
     use super::Prompt;
     use crate::error::FaultSource;
     use crate::prompt::error::{NewPromptError, NewPromptErrorKind};
+    use crate::prompt::truncate;
 
     #[test]
     fn default_template() {
@@ -116,18 +188,18 @@ mod test {
 
     #[test]
     fn empty_template() {
-        Prompt::new("".into()).unwrap();
+        Prompt::new("".into(), None).unwrap();
     }
 
     #[test]
     fn template_ok() {
-        Prompt::new("{{doc.title}}: {{doc.overview}}".into()).unwrap();
+        Prompt::new("{{doc.title}}: {{doc.overview}}".into(), None).unwrap();
     }
 
     #[test]
     fn template_syntax() {
         assert!(matches!(
-            Prompt::new("{{doc.title: {{doc.overview}}".into()),
+            Prompt::new("{{doc.title: {{doc.overview}}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::CannotParseTemplate(_),
                 fault: FaultSource::User
@@ -138,7 +210,7 @@ mod test {
     #[test]
     fn template_missing_doc() {
         assert!(matches!(
-            Prompt::new("{{title}}: {{overview}}".into()),
+            Prompt::new("{{title}}: {{overview}}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::InvalidFieldsInTemplate(_),
                 fault: FaultSource::User
@@ -148,29 +220,62 @@ mod test {
 
     #[test]
     fn template_nested_doc() {
-        Prompt::new("{{doc.actor.firstName}}: {{doc.actor.lastName}}".into()).unwrap();
+        Prompt::new("{{doc.actor.firstName}}: {{doc.actor.lastName}}".into(), None).unwrap();
     }
 
     #[test]
     fn template_fields() {
-        Prompt::new("{% for field in fields %}{{field}}{% endfor %}".into()).unwrap();
+        Prompt::new("{% for field in fields %}{{field}}{% endfor %}".into(), None).unwrap();
     }
 
     #[test]
     fn template_fields_ok() {
-        Prompt::new("{% for field in fields %}{{field.name}}: {{field.value}}{% endfor %}".into())
-            .unwrap();
+        Prompt::new(
+            "{% for field in fields %}{{field.name}}: {{field.value}}{% endfor %}".into(),
+            None,
+        )
+        .unwrap();
     }
 
     #[test]
     fn template_fields_invalid() {
         assert!(matches!(
             // intentionally garbled field
-            Prompt::new("{% for field in fields %}{{field.vaelu}} {% endfor %}".into()),
+            Prompt::new("{% for field in fields %}{{field.vaelu}} {% endfor %}".into(), None),
             Err(NewPromptError {
                 kind: NewPromptErrorKind::InvalidFieldsInTemplate(_),
                 fault: FaultSource::User
             })
         ));
     }
+
+    // todo: test truncation
+    #[test]
+    fn template_truncation() {
+        let mut s = "インテル ザー ビーグル".to_string();
+
+        truncate(&mut s, 42);
+        assert_eq!(s, "インテル ザー ビーグル");
+
+        assert_eq!(s.len(), 32);
+        truncate(&mut s, 32);
+        assert_eq!(s, "インテル ザー ビーグル");
+
+        truncate(&mut s, 31);
+        assert_eq!(s, "インテル ザー ビーグ");
+        truncate(&mut s, 30);
+        assert_eq!(s, "インテル ザー ビーグ");
+        truncate(&mut s, 28);
+        assert_eq!(s, "インテル ザー ビー");
+        truncate(&mut s, 26);
+        assert_eq!(s, "インテル ザー ビー");
+        truncate(&mut s, 25);
+        assert_eq!(s, "インテル ザー ビ");
+
+        assert_eq!("イ".len(), 3);
+        truncate(&mut s, 3);
+        assert_eq!(s, "イ");
+        truncate(&mut s, 2);
+        assert_eq!(s, "");
+    }
 }

milli/src/search/facet/facet_distribution.rs

@@ -1,4 +1,5 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
+use std::fmt::Display;
 use std::ops::ControlFlow;
 use std::{fmt, mem};
 
@@ -37,6 +38,15 @@ pub enum OrderBy {
     Count,
 }
 
+impl Display for OrderBy {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            OrderBy::Lexicographic => f.write_str("alphabetically"),
+            OrderBy::Count => f.write_str("by count"),
+        }
+    }
+}
+
 pub struct FacetDistribution<'a> {
     facets: Option<HashMap<String, OrderBy>>,
     candidates: Option<RoaringBitmap>,
@@ -100,7 +110,6 @@ impl<'a> FacetDistribution<'a> {
                 let mut lexicographic_distribution = BTreeMap::new();
                 let mut key_buffer: Vec<_> = field_id.to_be_bytes().to_vec();
 
-                let distribution_prelength = distribution.len();
                 let db = self.index.field_id_docid_facet_f64s;
                 for docid in candidates {
                     key_buffer.truncate(mem::size_of::<FieldId>());
@@ -113,23 +122,21 @@ impl<'a> FacetDistribution<'a> {
                     for result in iter {
                         let ((_, _, value), ()) = result?;
                         *lexicographic_distribution.entry(value.to_string()).or_insert(0) += 1;
-
-                        if lexicographic_distribution.len() - distribution_prelength
-                            == self.max_values_per_facet
-                        {
-                            break;
-                        }
                     }
                 }
 
-                distribution.extend(lexicographic_distribution);
+                distribution.extend(
+                    lexicographic_distribution
+                        .into_iter()
+                        .take(self.max_values_per_facet.saturating_sub(distribution.len())),
+                );
             }
             FacetType::String => {
                 let mut normalized_distribution = BTreeMap::new();
                 let mut key_buffer: Vec<_> = field_id.to_be_bytes().to_vec();
 
                 let db = self.index.field_id_docid_facet_strings;
-                'outer: for docid in candidates {
+                for docid in candidates {
                     key_buffer.truncate(mem::size_of::<FieldId>());
                     key_buffer.extend_from_slice(&docid.to_be_bytes());
                     let iter = db
@@ -144,14 +151,14 @@ impl<'a> FacetDistribution<'a> {
                             .or_insert_with(|| (original_value, 0));
                         *count += 1;
 
-                        if normalized_distribution.len() == self.max_values_per_facet {
-                            break 'outer;
-                        }
+                        // we'd like to break here if we have enough facet values, but we are collecting them by increasing docid,
+                        // so higher ranked facets could be in later docids
                     }
                 }
 
                 let iter = normalized_distribution
                     .into_iter()
+                    .take(self.max_values_per_facet.saturating_sub(distribution.len()))
                     .map(|(_normalized, (original, count))| (original.to_string(), count));
                 distribution.extend(iter);
             }
@@ -467,7 +474,7 @@ mod tests {
             .execute()
             .unwrap();
 
-        milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 1}}"###);
+        milli_snap!(format!("{map:?}"), @r###"{"colour": {"Blue": 2}}"###);
 
         let map = FacetDistribution::new(&txn, &index)
             .facets(iter::once(("colour", OrderBy::Count)))

milli/src/search/facet/filter.rs

@@ -12,7 +12,7 @@ use serde_json::Value;
 use super::facet_range_search;
 use crate::error::{Error, UserError};
 use crate::heed_codec::facet::{
-    FacetGroupKey, FacetGroupKeyCodec, FacetGroupValueCodec, OrderedF64Codec,
+    FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, OrderedF64Codec,
 };
 use crate::index::db_name::FACET_ID_STRING_DOCIDS;
 use crate::{
@@ -75,25 +75,21 @@ impl<'a> Display for FilterError<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::AttributeNotFilterable { attribute, filterable_fields } => {
+                write!(f, "Attribute `{attribute}` is not filterable.")?;
                 if filterable_fields.is_empty() {
-                    write!(
-                        f,
-                        "Attribute `{}` is not filterable. This index does not have configured filterable attributes.",
-                        attribute,
-                    )
+                    write!(f, " This index does not have configured filterable attributes.")
                 } else {
-                    let filterables_list = filterable_fields
-                        .iter()
-                        .map(AsRef::as_ref)
-                        .collect::<Vec<&str>>()
-                        .join(" ");
-
-                    write!(
-                        f,
-                        "Attribute `{}` is not filterable. Available filterable attributes are: `{}`.",
-                        attribute,
-                        filterables_list,
-                    )
+                    write!(f, " Available filterable attributes are: ")?;
+                    let mut filterables_list =
+                        filterable_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>();
+                    filterables_list.sort_unstable();
+                    for (idx, filterable) in filterables_list.iter().enumerate() {
+                        write!(f, "`{filterable}`")?;
+                        if idx != filterables_list.len() - 1 {
+                            write!(f, ", ")?;
+                        }
+                    }
+                    write!(f, ".")
                 }
             }
             Self::TooDeep => write!(
@@ -340,6 +336,24 @@ impl<'a> Filter<'a> {
 
                 return Ok(docids);
             }
+            Condition::StartsWith { keyword: _, word } => {
+                let value = crate::normalize_facet(word.value());
+                let base = FacetGroupKey { field_id, level: 0, left_bound: value.as_str() };
+                let docids = strings_db
+                    .prefix_iter(rtxn, &base)?
+                    .map(|result| -> Result<RoaringBitmap> {
+                        match result {
+                            Ok((_facet_group_key, FacetGroupValue { bitmap, .. })) => Ok(bitmap),
+                            Err(_e) => Err(InternalError::from(SerializationError::Decoding {
+                                db_name: Some(FACET_ID_STRING_DOCIDS),
+                            })
+                            .into()),
+                        }
+                    })
+                    .union()?;
+
+                return Ok(docids);
+            }
         };
 
         let mut output = RoaringBitmap::new();

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -15,14 +15,14 @@ use serde_json::Value;
 use super::helpers::{create_writer, writer_into_reader, GrenadParameters};
 use crate::error::FaultSource;
 use crate::index::IndexEmbeddingConfig;
-use crate::prompt::Prompt;
+use crate::prompt::{FieldsIdsMapWithMetadata, Prompt};
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::settings::InnerIndexSettingsDiff;
 use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution};
 use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState, RESERVED_VECTORS_FIELD_NAME};
 use crate::vector::settings::{EmbedderAction, ReindexAction};
 use crate::vector::{Embedder, Embeddings};
-use crate::{try_split_array_at, DocumentId, FieldId, FieldsIdsMap, Result, ThreadPoolNoAbort};
+use crate::{try_split_array_at, DocumentId, FieldId, Result, ThreadPoolNoAbort};
 
 /// The length of the elements that are always in the buffer when inserting new values.
 const TRUNCATE_SIZE: usize = size_of::<DocumentId>();
@@ -189,7 +189,13 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     let reindex_vectors = settings_diff.reindex_vectors();
 
     let old_fields_ids_map = &settings_diff.old.fields_ids_map;
+    let old_fields_ids_map =
+        FieldsIdsMapWithMetadata::new(old_fields_ids_map, &settings_diff.old.searchable_fields_ids);
+
     let new_fields_ids_map = &settings_diff.new.fields_ids_map;
+    let new_fields_ids_map =
+        FieldsIdsMapWithMetadata::new(new_fields_ids_map, &settings_diff.new.searchable_fields_ids);
+
     // the vector field id may have changed
     let old_vectors_fid = old_fields_ids_map.id(RESERVED_VECTORS_FIELD_NAME);
 
@@ -376,7 +382,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                             );
                             continue;
                         }
-                        regenerate_prompt(obkv, prompt, new_fields_ids_map)?
+                        regenerate_prompt(obkv, prompt, &new_fields_ids_map)?
                     }
                 },
                 // prompt regeneration is only triggered for existing embedders
@@ -393,7 +399,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                         regenerate_if_prompt_changed(
                             obkv,
                             (old_prompt, prompt),
-                            (old_fields_ids_map, new_fields_ids_map),
+                            (&old_fields_ids_map, &new_fields_ids_map),
                         )?
                     } else {
                         // we can simply ignore user provided vectors as they are not regenerated and are
@@ -409,7 +415,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                     prompt,
                     (add_to_user_provided, remove_from_user_provided),
                     (old, new),
-                    (old_fields_ids_map, new_fields_ids_map),
+                    (&old_fields_ids_map, &new_fields_ids_map),
                     document_id,
                     embedder_name,
                     embedder_is_manual,
@@ -479,7 +485,10 @@ fn extract_vector_document_diff(
     prompt: &Prompt,
     (add_to_user_provided, remove_from_user_provided): (&mut RoaringBitmap, &mut RoaringBitmap),
     (old, new): (VectorState, VectorState),
-    (old_fields_ids_map, new_fields_ids_map): (&FieldsIdsMap, &FieldsIdsMap),
+    (old_fields_ids_map, new_fields_ids_map): (
+        &FieldsIdsMapWithMetadata,
+        &FieldsIdsMapWithMetadata,
+    ),
     document_id: impl Fn() -> Value,
     embedder_name: &str,
     embedder_is_manual: bool,
@@ -599,7 +608,10 @@ fn extract_vector_document_diff(
 fn regenerate_if_prompt_changed(
     obkv: obkv::KvReader<'_, FieldId>,
     (old_prompt, new_prompt): (&Prompt, &Prompt),
-    (old_fields_ids_map, new_fields_ids_map): (&FieldsIdsMap, &FieldsIdsMap),
+    (old_fields_ids_map, new_fields_ids_map): (
+        &FieldsIdsMapWithMetadata,
+        &FieldsIdsMapWithMetadata,
+    ),
 ) -> Result<VectorStateDelta> {
     let old_prompt =
         old_prompt.render(obkv, DelAdd::Deletion, old_fields_ids_map).unwrap_or(Default::default());
@@ -614,7 +626,7 @@ fn regenerate_if_prompt_changed(
 fn regenerate_prompt(
     obkv: obkv::KvReader<'_, FieldId>,
     prompt: &Prompt,
-    new_fields_ids_map: &FieldsIdsMap,
+    new_fields_ids_map: &FieldsIdsMapWithMetadata,
 ) -> Result<VectorStateDelta> {
     let prompt = prompt.render(obkv, DelAdd::Addition, new_fields_ids_map)?;
 

milli/src/update/index_documents/extract/mod.rs

@@ -34,7 +34,7 @@ use crate::index::IndexEmbeddingConfig;
 use crate::update::settings::InnerIndexSettingsDiff;
 use crate::vector::error::PossibleEmbeddingMistakes;
 use crate::{FieldId, Result, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder};
-
+/// Hello!
 /// Extract data for each databases from obkv documents in parallel.
 /// Send data in grenad file over provided Sender.
 #[allow(clippy::too_many_arguments)]

milli/src/update/index_documents/mod.rs

@@ -2740,6 +2740,7 @@ mod tests {
                         api_key: Setting::NotSet,
                         dimensions: Setting::Set(3),
                         document_template: Setting::NotSet,
+                        document_template_max_bytes: Setting::NotSet,
                         url: Setting::NotSet,
                         request: Setting::NotSet,
                         response: Setting::NotSet,

milli/src/update/settings.rs

@@ -1,5 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::convert::TryInto;
+use std::num::NonZeroUsize;
 use std::result::Result as StdResult;
 use std::sync::Arc;
 
@@ -19,6 +20,7 @@ use crate::index::{
     IndexEmbeddingConfig, DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS,
 };
 use crate::order_by_map::OrderByMap;
+use crate::prompt::default_max_bytes;
 use crate::proximity::ProximityPrecision;
 use crate::update::index_documents::IndexDocumentsMethod;
 use crate::update::{IndexDocuments, UpdateIndexingStep};
@@ -1238,7 +1240,7 @@ impl InnerIndexSettingsDiff {
         old_settings: InnerIndexSettings,
         new_settings: InnerIndexSettings,
         primary_key_id: Option<FieldId>,
-        embedding_config_updates: BTreeMap<String, EmbedderAction>,
+        mut embedding_config_updates: BTreeMap<String, EmbedderAction>,
         settings_update_only: bool,
     ) -> Self {
         let only_additional_fields = match (
@@ -1273,6 +1275,32 @@ impl InnerIndexSettingsDiff {
         let cache_user_defined_searchables = old_settings.user_defined_searchable_fields
             != new_settings.user_defined_searchable_fields;
 
+        // if the user-defined searchables changed, then we need to reindex prompts.
+        if cache_user_defined_searchables {
+            for (embedder_name, (config, _)) in new_settings.embedding_configs.inner_as_ref() {
+                // skip embedders that don't use document templates
+                if !config.uses_document_template() {
+                    continue;
+                }
+
+                // note: this could currently be entry.or_insert(..), but we're future-proofing with an explicit match
+                // this always makes the code clearer by explicitly handling the cases
+                match embedding_config_updates.entry(embedder_name.clone()) {
+                    std::collections::btree_map::Entry::Vacant(entry) => {
+                        entry.insert(EmbedderAction::Reindex(ReindexAction::RegeneratePrompts));
+                    }
+                    std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
+                        EmbedderAction::WriteBackToDocuments(_) => { /* we are deleting this embedder, so no point in regeneration */
+                        }
+                        EmbedderAction::Reindex(ReindexAction::FullReindex) => { /* we are already fully reindexing */
+                        }
+                        EmbedderAction::Reindex(ReindexAction::RegeneratePrompts) => { /* we are already regenerating prompts */
+                        }
+                    },
+                };
+            }
+        }
+
         InnerIndexSettingsDiff {
             old: old_settings,
             new: new_settings,
@@ -1547,16 +1575,30 @@ fn validate_prompt(
             api_key,
             dimensions,
             document_template: Setting::Set(template),
+            document_template_max_bytes,
             url,
             request,
             response,
             distribution,
             headers,
         }) => {
+            let max_bytes = match document_template_max_bytes.set() {
+                Some(max_bytes) => NonZeroUsize::new(max_bytes).ok_or_else(|| {
+                    crate::error::UserError::InvalidSettingsDocumentTemplateMaxBytes {
+                        embedder_name: name.to_owned(),
+                    }
+                })?,
+                None => default_max_bytes(),
+            };
+
             // validate
-            let template = crate::prompt::Prompt::new(template)
-                .map(|prompt| crate::prompt::PromptData::from(prompt).template)
-                .map_err(|inner| UserError::InvalidPromptForEmbeddings(name.to_owned(), inner))?;
+            let template = crate::prompt::Prompt::new(
+                template,
+                // always specify a max_bytes
+                Some(max_bytes),
+            )
+            .map(|prompt| crate::prompt::PromptData::from(prompt).template)
+            .map_err(|inner| UserError::InvalidPromptForEmbeddings(name.to_owned(), inner))?;
 
             Ok(Setting::Set(EmbeddingSettings {
                 source,
@@ -1565,6 +1607,7 @@ fn validate_prompt(
                 api_key,
                 dimensions,
                 document_template: Setting::Set(template),
+                document_template_max_bytes,
                 url,
                 request,
                 response,
@@ -1589,6 +1632,7 @@ pub fn validate_embedding_settings(
         api_key,
         dimensions,
         document_template,
+        document_template_max_bytes,
         url,
         request,
         response,
@@ -1628,6 +1672,7 @@ pub fn validate_embedding_settings(
             api_key,
             dimensions,
             document_template,
+            document_template_max_bytes,
             url,
             request,
             response,
@@ -1700,6 +1745,12 @@ pub fn validate_embedding_settings(
                 inferred_source,
                 name,
             )?;
+            check_unset(
+                &document_template_max_bytes,
+                EmbeddingSettings::DOCUMENT_TEMPLATE_MAX_BYTES,
+                inferred_source,
+                name,
+            )?;
             check_set(&dimensions, EmbeddingSettings::DIMENSIONS, inferred_source, name)?;
 
             check_unset(&url, EmbeddingSettings::URL, inferred_source, name)?;
@@ -1722,6 +1773,7 @@ pub fn validate_embedding_settings(
         api_key,
         dimensions,
         document_template,
+        document_template_max_bytes,
         url,
         request,
         response,

milli/src/vector/mod.rs

@@ -305,6 +305,16 @@ impl Embedder {
             Embedder::Rest(embedder) => embedder.distribution(),
         }
     }
+
+    pub fn uses_document_template(&self) -> bool {
+        match self {
+            Embedder::HuggingFace(_)
+            | Embedder::OpenAi(_)
+            | Embedder::Ollama(_)
+            | Embedder::Rest(_) => true,
+            Embedder::UserProvided(_) => false,
+        }
+    }
 }
 
 /// Describes the mean and sigma of distribution of embedding similarity in the embedding space.

milli/src/vector/openai.rs

@@ -66,11 +66,11 @@ pub enum EmbeddingModel {
     // # WARNING
     //
     // If ever adding a model, make sure to add it to the list of supported models below.
-    #[default]
     #[serde(rename = "text-embedding-ada-002")]
     #[deserr(rename = "text-embedding-ada-002")]
     TextEmbeddingAda002,
 
+    #[default]
     #[serde(rename = "text-embedding-3-small")]
     #[deserr(rename = "text-embedding-3-small")]
     TextEmbedding3Small,

milli/src/vector/settings.rs

@@ -1,11 +1,12 @@
 use std::collections::BTreeMap;
+use std::num::NonZeroUsize;
 
 use deserr::Deserr;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
 
 use super::{ollama, openai, DistributionShift};
-use crate::prompt::PromptData;
+use crate::prompt::{default_max_bytes, PromptData};
 use crate::update::Setting;
 use crate::vector::EmbeddingConfig;
 use crate::UserError;
@@ -34,6 +35,9 @@ pub struct EmbeddingSettings {
     pub document_template: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    pub document_template_max_bytes: Setting<usize>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default)]
     pub url: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
@@ -111,6 +115,7 @@ impl SettingsDiff {
                     mut response,
                     mut distribution,
                     mut headers,
+                    mut document_template_max_bytes,
                 } = old;
 
                 let EmbeddingSettings {
@@ -125,6 +130,7 @@ impl SettingsDiff {
                     response: new_response,
                     distribution: new_distribution,
                     headers: new_headers,
+                    document_template_max_bytes: new_document_template_max_bytes,
                 } = new;
 
                 let mut reindex_action = None;
@@ -142,6 +148,7 @@ impl SettingsDiff {
                         &mut request,
                         &mut response,
                         &mut document_template,
+                        &mut document_template_max_bytes,
                         &mut headers,
                     )
                 }
@@ -190,6 +197,23 @@ impl SettingsDiff {
                     );
                 }
 
+                if document_template_max_bytes.apply(new_document_template_max_bytes) {
+                    let previous_document_template_max_bytes =
+                        document_template_max_bytes.set().unwrap_or(default_max_bytes().get());
+                    let new_document_template_max_bytes =
+                        new_document_template_max_bytes.set().unwrap_or(default_max_bytes().get());
+
+                    // only reindex if the size increased. Reasoning:
+                    // - size decrease is a performance optimization, so we don't reindex and we keep the more accurate vectors
+                    // - size increase is an accuracy optimization, so we want to reindex
+                    if new_document_template_max_bytes > previous_document_template_max_bytes {
+                        ReindexAction::push_action(
+                            &mut reindex_action,
+                            ReindexAction::RegeneratePrompts,
+                        )
+                    }
+                }
+
                 distribution.apply(new_distribution);
                 api_key.apply(new_api_key);
                 headers.apply(new_headers);
@@ -206,6 +230,7 @@ impl SettingsDiff {
                     response,
                     distribution,
                     headers,
+                    document_template_max_bytes,
                 };
 
                 match reindex_action {
@@ -239,6 +264,7 @@ fn apply_default_for_source(
     request: &mut Setting<serde_json::Value>,
     response: &mut Setting<serde_json::Value>,
     document_template: &mut Setting<String>,
+    document_template_max_bytes: &mut Setting<usize>,
     headers: &mut Setting<BTreeMap<String, String>>,
 ) {
     match source {
@@ -286,6 +312,7 @@ fn apply_default_for_source(
             *request = Setting::NotSet;
             *response = Setting::NotSet;
             *document_template = Setting::NotSet;
+            *document_template_max_bytes = Setting::NotSet;
             *headers = Setting::NotSet;
         }
         Setting::NotSet => {}
@@ -316,6 +343,7 @@ impl EmbeddingSettings {
     pub const API_KEY: &'static str = "apiKey";
     pub const DIMENSIONS: &'static str = "dimensions";
     pub const DOCUMENT_TEMPLATE: &'static str = "documentTemplate";
+    pub const DOCUMENT_TEMPLATE_MAX_BYTES: &'static str = "documentTemplateMaxBytes";
 
     pub const URL: &'static str = "url";
     pub const REQUEST: &'static str = "request";
@@ -459,6 +487,8 @@ impl std::fmt::Display for EmbedderSource {
 impl From<EmbeddingConfig> for EmbeddingSettings {
     fn from(value: EmbeddingConfig) -> Self {
         let EmbeddingConfig { embedder_options, prompt } = value;
+        let document_template_max_bytes =
+            Setting::Set(prompt.max_bytes.unwrap_or(default_max_bytes()).get());
         match embedder_options {
             super::EmbedderOptions::HuggingFace(super::hf::EmbedderOptions {
                 model,
@@ -471,6 +501,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::NotSet,
                 dimensions: Setting::NotSet,
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::NotSet,
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -490,6 +521,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::some_or_not_set(url),
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -509,6 +541,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::some_or_not_set(url),
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -525,6 +558,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::NotSet,
                 dimensions: Setting::Set(dimensions),
                 document_template: Setting::NotSet,
+                document_template_max_bytes: Setting::NotSet,
                 url: Setting::NotSet,
                 request: Setting::NotSet,
                 response: Setting::NotSet,
@@ -546,6 +580,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 api_key: Setting::some_or_not_set(api_key),
                 dimensions: Setting::some_or_not_set(dimensions),
                 document_template: Setting::Set(prompt.template),
+                document_template_max_bytes,
                 url: Setting::Set(url),
                 request: Setting::Set(request),
                 response: Setting::Set(response),
@@ -566,6 +601,7 @@ impl From<EmbeddingSettings> for EmbeddingConfig {
             api_key,
             dimensions,
             document_template,
+            document_template_max_bytes,
             url,
             request,
             response,
@@ -648,7 +684,12 @@ impl From<EmbeddingSettings> for EmbeddingConfig {
         }
 
         if let Setting::Set(template) = document_template {
-            this.prompt = PromptData { template }
+            let max_bytes = document_template_max_bytes
+                .set()
+                .and_then(NonZeroUsize::new)
+                .unwrap_or(default_max_bytes());
+
+            this.prompt = PromptData { template, max_bytes: Some(max_bytes) }
         }
 
         this