Change base index size to 2TiB

Co-authored-by: Clément Renault <clement@meilisearch.com>

.github/scripts/check-release.sh

@@ -3,7 +3,7 @@
 # check_tag $current_tag $file_tag $file_name
 function check_tag {
   if [[ "$1" != "$2" ]]; then
-      echo "Error: the current tag does not match the version in Cargo.toml: found $2 - expected $1"
+      echo "Error: the current tag does not match the version in $3: found $2 - expected $1"
       ret=1
   fi
 }
@@ -11,8 +11,12 @@ function check_tag {
 ret=0
 current_tag=${GITHUB_REF#'refs/tags/v'}
 
-file_tag="$(grep '^version = ' Cargo.toml | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
-check_tag $current_tag $file_tag
+toml_files='*/Cargo.toml'
+for toml_file in $toml_files;
+do
+    file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+    check_tag $current_tag $file_tag $toml_file
+done
 
 lock_file='Cargo.lock'
 lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')

.github/workflows/update-cargo-toml-version.yml

@@ -29,7 +29,7 @@ jobs:
         run: |
           raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
           new_string="version = \"$raw_new_version\""
-          sd '^version = "\d+.\d+.\w+"$' "$new_string" Cargo.toml
+          sd '^version = "\d+.\d+.\w+"$' "$new_string" */Cargo.toml
       - name: Build Meilisearch to update Cargo.lock
         run: cargo build
       - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch

Cargo.lock

@@ -609,9 +609,9 @@ dependencies = [
 
 [[package]]
 name = "cargo_toml"
-version = "0.14.1"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bfbc36312494041e2cdd5f06697b7e89d4b76f42773a0b5556ac290ff22acc2"
+checksum = "497049e9477329f8f6a559972ee42e117487d01d1e8c2cc9f836ea6fa23a9e1a"
 dependencies = [
  "serde",
  "toml",
@@ -1113,9 +1113,9 @@ dependencies = [
 
 [[package]]
 name = "deserr"
-version = "0.5.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c71c14985c842bf1e520b1ebcd22daff6aeece32f510e11f063cecf9b308c04b"
+checksum = "6eee2844f21cf7fb5693aae1fb8f1658127acfdb2fc072167d68a9152584ae64"
 dependencies = [
  "actix-http",
  "actix-utils",
@@ -1130,9 +1130,9 @@ dependencies = [
 
 [[package]]
 name = "deserr-internal"
-version = "0.5.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cae1c51b191528c9e4e5d6cff671de94f61fcda1c206cc891251e0cf438c941a"
+checksum = "c27246f8ca9eeba9dd70d614b664dc43b529251ed7bd9e633131010d340da4b9"
 dependencies = [
  "convert_case 0.5.0",
  "proc-macro2",

Cargo.toml

@@ -16,15 +16,6 @@ members = [
     "benchmarks"
 ]
 
-[workspace.package]
-version = "1.0.0"
-authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
-description = "Meilisearch HTTP server"
-homepage = "https://meilisearch.com"
-readme = "README.md"
-edition = "2021"
-license = "MIT"
-
 [profile.release]
 codegen-units = 1
 

Dockerfile

@@ -1,5 +1,7 @@
 # Compile
-FROM    rust:bullseye AS compiler
+FROM    rust:alpine3.16 AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
 
 WORKDIR /meilisearch
 
@@ -11,20 +13,20 @@ ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
 RUN     set -eux; \
-        arch="$(dpkg --print-architecture)"; \
-        if [ "$arch" = "aarch64" ]; then \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
         cargo build --release
 
 # Run
-FROM    debian:11.6
+FROM    alpine:3.16
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apt update -q \
-        && apt install -q -y tini
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
 
 # add meilisearch to the `/bin` so you can run it from anywhere and it's easy
 # to find.

benchmarks/Cargo.toml

@@ -1,15 +1,9 @@
 [package]
 name = "benchmarks"
+version = "1.0.0"
+edition = "2018"
 publish = false
 
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
 [dependencies]
 anyhow = "1.0.65"
 csv = "1.1.6"

dump/Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "dump"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-edition.workspace = true
-homepage.workspace = true
-readme.workspace = true
-license.workspace = true
+version = "1.0.0"
+edition = "2021"
 
 [dependencies]
 anyhow = "1.0.65"

file-store/Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "file-store"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.0"
+edition = "2021"
 
 [dependencies]
 tempfile = "3.3.0"

filter-parser/Cargo.toml

@@ -1,16 +1,10 @@
 [package]
 name = "filter-parser"
+version = "1.0.0"
+edition = "2021"
 description = "The parser for the Meilisearch filter syntax"
 publish = false
 
-version.workspace = true
-authors.workspace = true
-# description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
 [dependencies]
 nom = "7.1.1"
 nom_locate = "4.0.0"

flatten-serde-json/Cargo.toml

@@ -1,17 +1,11 @@
 [package]
 name = "flatten-serde-json"
+version = "1.0.0"
+edition = "2021"
 description = "Flatten serde-json objects like elastic search"
 readme = "README.md"
 publish = false
 
-version.workspace = true
-authors.workspace = true
-# description.workspace = true
-homepage.workspace = true
-# readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
 [dependencies]
 serde_json = "1.0"
 

index-scheduler/Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "index-scheduler"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.0"
+edition = "2021"
 
 [dependencies]
 anyhow = "1.0.64"

index-scheduler/src/autobatcher.rs

@@ -88,11 +88,11 @@ pub enum BatchKind {
     DocumentClear {
         ids: Vec<TaskId>,
     },
-    DocumentOperation {
+    DocumentImport {
         method: IndexDocumentsMethod,
         allow_index_creation: bool,
         primary_key: Option<String>,
-        operation_ids: Vec<TaskId>,
+        import_ids: Vec<TaskId>,
     },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
@@ -102,12 +102,12 @@ pub enum BatchKind {
         allow_index_creation: bool,
         settings_ids: Vec<TaskId>,
     },
-    SettingsAndDocumentOperation {
+    SettingsAndDocumentImport {
         settings_ids: Vec<TaskId>,
         method: IndexDocumentsMethod,
         allow_index_creation: bool,
         primary_key: Option<String>,
-        operation_ids: Vec<TaskId>,
+        import_ids: Vec<TaskId>,
     },
     Settings {
         allow_index_creation: bool,
@@ -131,9 +131,9 @@ impl BatchKind {
     #[rustfmt::skip]
     fn allow_index_creation(&self) -> Option<bool> {
         match self {
-            BatchKind::DocumentOperation { allow_index_creation, .. }
+            BatchKind::DocumentImport { allow_index_creation, .. }
             | BatchKind::ClearAndSettings { allow_index_creation, .. }
-            | BatchKind::SettingsAndDocumentOperation { allow_index_creation, .. }
+            | BatchKind::SettingsAndDocumentImport { allow_index_creation, .. }
             | BatchKind::Settings { allow_index_creation, .. } => Some(*allow_index_creation),
             _ => None,
         }
@@ -141,8 +141,8 @@ impl BatchKind {
 
     fn primary_key(&self) -> Option<Option<&str>> {
         match self {
-            BatchKind::DocumentOperation { primary_key, .. }
-            | BatchKind::SettingsAndDocumentOperation { primary_key, .. } => {
+            BatchKind::DocumentImport { primary_key, .. }
+            | BatchKind::SettingsAndDocumentImport { primary_key, .. } => {
                 Some(primary_key.as_deref())
             }
             _ => None,
@@ -173,22 +173,22 @@ impl BatchKind {
                 if primary_key.is_none() || pk.is_none() || primary_key == pk.as_deref() =>
             {
                 (
-                    Continue(BatchKind::DocumentOperation {
+                    Continue(BatchKind::DocumentImport {
                         method,
                         allow_index_creation,
                         primary_key: pk,
-                        operation_ids: vec![task_id],
+                        import_ids: vec![task_id],
                     }),
                     allow_index_creation,
                 )
             }
             // if the primary key set in the task was different than ours we should stop and make this batch fail asap.
             K::DocumentImport { method, allow_index_creation, primary_key } => (
-                Break(BatchKind::DocumentOperation {
+                Break(BatchKind::DocumentImport {
                     method,
                     allow_index_creation,
                     primary_key,
-                    operation_ids: vec![task_id],
+                    import_ids: vec![task_id],
                 }),
                 allow_index_creation,
             ),
@@ -249,7 +249,7 @@ impl BatchKind {
             (
                 BatchKind::DocumentClear { mut ids }
                 | BatchKind::DocumentDeletion { deletion_ids: mut ids }
-                | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
+                | BatchKind::DocumentImport { method: _, allow_index_creation: _, primary_key: _, import_ids: mut ids }
                 | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
                 K::IndexDeletion,
             ) => {
@@ -258,7 +258,7 @@ impl BatchKind {
             }
             (
                 BatchKind::ClearAndSettings { settings_ids: mut ids, allow_index_creation: _, mut other }
-                | BatchKind::SettingsAndDocumentOperation { operation_ids: mut ids, method: _, allow_index_creation: _, primary_key: _, settings_ids: mut other },
+                | BatchKind::SettingsAndDocumentImport { import_ids: mut ids, method: _, allow_index_creation: _, primary_key: _, settings_ids: mut other },
                 K::IndexDeletion,
             ) => {
                 ids.push(id);
@@ -278,108 +278,63 @@ impl BatchKind {
                 K::DocumentImport { .. } | K::Settings { .. },
             ) => Break(this),
             (
-                BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, mut operation_ids },
+                BatchKind::DocumentImport { method: _, allow_index_creation: _, primary_key: _, import_ids: mut ids },
                 K::DocumentClear,
             ) => {
-                operation_ids.push(id);
-                Continue(BatchKind::DocumentClear { ids: operation_ids })
+                ids.push(id);
+                Continue(BatchKind::DocumentClear { ids })
             }
 
             // we can autobatch the same kind of document additions / updates
             (
-                BatchKind::DocumentOperation { method: ReplaceDocuments, allow_index_creation, primary_key: _, mut operation_ids },
+                BatchKind::DocumentImport { method: ReplaceDocuments, allow_index_creation, primary_key: _, mut import_ids },
                 K::DocumentImport { method: ReplaceDocuments, primary_key: pk, .. },
             ) => {
-                operation_ids.push(id);
-                Continue(BatchKind::DocumentOperation {
+                import_ids.push(id);
+                Continue(BatchKind::DocumentImport {
                     method: ReplaceDocuments,
                     allow_index_creation,
-                    operation_ids,
+                    import_ids,
                     primary_key: pk,
                 })
             }
             (
-                BatchKind::DocumentOperation { method: UpdateDocuments, allow_index_creation, primary_key: _, mut operation_ids },
+                BatchKind::DocumentImport { method: UpdateDocuments, allow_index_creation, primary_key: _, mut import_ids },
                 K::DocumentImport { method: UpdateDocuments, primary_key: pk, .. },
             ) => {
-                operation_ids.push(id);
-                Continue(BatchKind::DocumentOperation {
+
+                import_ids.push(id);
+                Continue(BatchKind::DocumentImport {
                     method: UpdateDocuments,
                     allow_index_creation,
                     primary_key: pk,
-                    operation_ids,
+                    import_ids,
                 })
             }
-            (
-                BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
-                K::DocumentDeletion,
-            ) => {
-                operation_ids.push(id);
 
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids,
-                })
-            }
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
-                this @ BatchKind::DocumentOperation { .. },
-                K::DocumentImport { .. },
+                this @ BatchKind::DocumentImport { .. },
+                K::DocumentDeletion | K::DocumentImport { .. },
             ) => Break(this),
 
             (
-                BatchKind::DocumentOperation { method, allow_index_creation, primary_key, operation_ids },
+                BatchKind::DocumentImport { method, allow_index_creation, primary_key, import_ids },
                 K::Settings { .. },
-            ) => Continue(BatchKind::SettingsAndDocumentOperation {
+            ) => Continue(BatchKind::SettingsAndDocumentImport {
                 settings_ids: vec![id],
                 method,
                 allow_index_creation,
                 primary_key,
-                operation_ids,
+                import_ids,
             }),
 
             (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentClear) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
-            // we can autobatch the deletion and import if the index already exists
-            (
-                BatchKind::DocumentDeletion { mut deletion_ids },
-                K::DocumentImport { method, allow_index_creation, primary_key }
-            ) if index_already_exists => {
-                deletion_ids.push(id);
-
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids: deletion_ids,
-                })
-            }
-            // we can autobatch the deletion and import if both can't create an index
-            (
-                BatchKind::DocumentDeletion { mut deletion_ids },
-                K::DocumentImport { method, allow_index_creation, primary_key }
-            ) if !allow_index_creation => {
-                deletion_ids.push(id);
-
-                Continue(BatchKind::DocumentOperation {
-                    method,
-                    allow_index_creation,
-                    primary_key,
-                    operation_ids: deletion_ids,
-                })
-            }
-            // we can't autobatch a deletion and an import if the index does not exists but would be created by an addition
-            (
-                this @ BatchKind::DocumentDeletion { .. },
-                K::DocumentImport { .. }
-            ) => {
-                Break(this)
-            }
+            (this @ BatchKind::DocumentDeletion { .. }, K::DocumentImport { .. }) => Break(this),
             (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentDeletion) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentDeletion { deletion_ids })
@@ -448,60 +403,60 @@ impl BatchKind {
                 })
             }
             (
-                BatchKind::SettingsAndDocumentOperation { settings_ids, method: _, mut operation_ids, allow_index_creation, primary_key: _ },
+                BatchKind::SettingsAndDocumentImport { settings_ids, method: _, import_ids: mut other, allow_index_creation, primary_key: _ },
                 K::DocumentClear,
             ) => {
-                operation_ids.push(id);
+                other.push(id);
                 Continue(BatchKind::ClearAndSettings {
                     settings_ids,
-                    other: operation_ids,
+                    other,
                     allow_index_creation,
                 })
             }
 
             (
-                BatchKind::SettingsAndDocumentOperation { settings_ids, method: ReplaceDocuments, mut operation_ids, allow_index_creation, primary_key: _},
+                BatchKind::SettingsAndDocumentImport { settings_ids, method: ReplaceDocuments, mut import_ids, allow_index_creation, primary_key: _},
                 K::DocumentImport { method: ReplaceDocuments, primary_key: pk2, .. },
             ) => {
-                operation_ids.push(id);
-                Continue(BatchKind::SettingsAndDocumentOperation {
+                import_ids.push(id);
+                Continue(BatchKind::SettingsAndDocumentImport {
                     settings_ids,
                     method: ReplaceDocuments,
                     allow_index_creation,
                         primary_key: pk2,
-                    operation_ids,
+                    import_ids,
                 })
             }
             (
-                BatchKind::SettingsAndDocumentOperation { settings_ids, method: UpdateDocuments, allow_index_creation, primary_key: _, mut operation_ids },
+                BatchKind::SettingsAndDocumentImport { settings_ids, method: UpdateDocuments, allow_index_creation, primary_key: _, mut import_ids },
                 K::DocumentImport { method: UpdateDocuments, primary_key: pk2, .. },
             ) => {
-                operation_ids.push(id);
-                Continue(BatchKind::SettingsAndDocumentOperation {
+                import_ids.push(id);
+                Continue(BatchKind::SettingsAndDocumentImport {
                     settings_ids,
                     method: UpdateDocuments,
                     allow_index_creation,
                     primary_key: pk2,
-                    operation_ids,
+                    import_ids,
                 })
             }
             // But we can't batch a settings and a doc op with another doc op
             // this MUST be AFTER the two previous branch
             (
-                this @ BatchKind::SettingsAndDocumentOperation { .. },
+                this @ BatchKind::SettingsAndDocumentImport { .. },
                 K::DocumentDeletion | K::DocumentImport { .. },
             ) => Break(this),
             (
-                BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
+                BatchKind::SettingsAndDocumentImport { mut settings_ids, method, allow_index_creation,primary_key, import_ids },
                 K::Settings { .. },
             ) => {
                 settings_ids.push(id);
-                Continue(BatchKind::SettingsAndDocumentOperation {
+                Continue(BatchKind::SettingsAndDocumentImport {
                     settings_ids,
                     method,
                     allow_index_creation,
                     primary_key,
-                    operation_ids,
+                    import_ids,
                 })
             }
             (
@@ -633,29 +588,29 @@ mod tests {
     fn autobatch_simple_operation_together() {
         // we can autobatch one or multiple `ReplaceDocuments` together.
         // if the index exists.
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, false , None), doc_imp(ReplaceDocuments, false , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1, 2] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, false , None), doc_imp(ReplaceDocuments, false , None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1, 2] }, false))");
 
         // if it doesn't exists.
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1, 2] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp( ReplaceDocuments, true , None), doc_imp(ReplaceDocuments, true , None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
 
         // we can autobatch one or multiple `UpdateDocuments` together.
         // if the index exists.
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1, 2] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1, 2] }, false))");
 
         // if it doesn't exists.
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1, 2] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1, 2] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple DocumentDeletion together
         debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
@@ -673,83 +628,56 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [settings(true), settings(true), settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0, 1, 2] }, true))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), settings(false), settings(false)]), @"Some((Settings { allow_index_creation: false, settings_ids: [0, 1, 2] }, false))");
-
-        // We can autobatch document addition with document deletion
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        // And the other way around
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
     }
 
     #[test]
     fn simple_document_operation_dont_autobatch_with_other() {
         // addition, updates and deletion can't batch together
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]
     fn document_addition_batch_with_settings() {
         // simple case
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
 
         // multiple settings and doc addition
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None), settings(true), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [2, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
 
         // addition and setting unordered
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 2] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 2] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1, 3], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 2] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1, 3], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 2] }, true))");
 
         // We ensure this kind of batch doesn't batch with forbidden operations
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_imp(UpdateDocuments, true, None)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_imp(ReplaceDocuments, true, None)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), doc_del()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_create()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_update()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), settings(true), idx_swap()]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: UpdateDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
     }
 
     #[test]
@@ -861,73 +789,67 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
         // The third and final case is when the first task doesn't create an index but is directly followed by a task creating an index. In this case we can't batch whith what
         // follows because we first need to process the erronous batch.
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(true), idx_del()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(true), idx_del()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(true), doc_clr(), idx_del()]), @"Some((DocumentImport { method: UpdateDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
     }
 
     #[test]
     fn allowed_and_disallowed_index_creation() {
         // `DocumentImport` can't be mixed with those disallowed to do so except if the index already exists.
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
 
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentOperation { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-
-        // batch deletion and addition
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0, 1] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, true, None), settings(true)]), @"Some((SettingsAndDocumentImport { settings_ids: [1], method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, import_ids: [0] }, false))");
     }
 
     #[test]
     fn autobatch_primary_key() {
         // ==> If I have a pk
         // With a single update
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
 
         // With a multiple updates
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0, 1] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0, 1] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other"))]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
 
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0, 1] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0, 1] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0, 1] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
 
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, Some("id"), [doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("other")), doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
 
         // ==> If I don't have a pk
         // With a single update
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("other"))]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("other"), import_ids: [0] }, true))"###);
 
         // With a multiple updates
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0, 1] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), operation_ids: [0] }, true))"###);
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0, 1] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(ReplaceDocuments, true, Some("id"))]), @"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, import_ids: [0] }, true))");
+        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("id")), doc_imp(ReplaceDocuments, true, None)]), @r###"Some((DocumentImport { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("id"), import_ids: [0] }, true))"###);
     }
 }

index-scheduler/src/batch.rs

@@ -28,7 +28,8 @@ use meilisearch_types::heed::{RoTxn, RwTxn};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
 use meilisearch_types::milli::heed::CompactionOption;
 use meilisearch_types::milli::update::{
-    DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod, Settings as MilliSettings,
+    DocumentAdditionResult, DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod,
+    Settings as MilliSettings,
 };
 use meilisearch_types::milli::{self, BEU32};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
@@ -85,21 +86,15 @@ pub(crate) enum Batch {
     },
 }
 
-#[derive(Debug)]
-pub(crate) enum DocumentOperation {
-    Add(Uuid),
-    Delete(Vec<String>),
-}
-
 /// A [batch](Batch) that combines multiple tasks operating on an index.
 #[derive(Debug)]
 pub(crate) enum IndexOperation {
-    DocumentOperation {
+    DocumentImport {
         index_uid: String,
         primary_key: Option<String>,
         method: IndexDocumentsMethod,
         documents_counts: Vec<u64>,
-        operations: Vec<DocumentOperation>,
+        content_files: Vec<Uuid>,
         tasks: Vec<Task>,
     },
     DocumentDeletion {
@@ -126,13 +121,13 @@ pub(crate) enum IndexOperation {
         settings: Vec<(bool, Settings<Unchecked>)>,
         settings_tasks: Vec<Task>,
     },
-    SettingsAndDocumentOperation {
+    SettingsAndDocumentImport {
         index_uid: String,
 
         primary_key: Option<String>,
         method: IndexDocumentsMethod,
         documents_counts: Vec<u64>,
-        operations: Vec<DocumentOperation>,
+        content_files: Vec<Uuid>,
         document_import_tasks: Vec<Task>,
 
         // The boolean indicates if it's a settings deletion or creation.
@@ -154,13 +149,13 @@ impl Batch {
                 tasks.iter().map(|task| task.uid).collect()
             }
             Batch::IndexOperation { op, .. } => match op {
-                IndexOperation::DocumentOperation { tasks, .. }
+                IndexOperation::DocumentImport { tasks, .. }
                 | IndexOperation::DocumentDeletion { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     tasks.iter().map(|task| task.uid).collect()
                 }
-                IndexOperation::SettingsAndDocumentOperation {
+                IndexOperation::SettingsAndDocumentImport {
                     document_import_tasks: tasks,
                     settings_tasks: other,
                     ..
@@ -195,12 +190,12 @@ impl Batch {
 impl IndexOperation {
     pub fn index_uid(&self) -> &str {
         match self {
-            IndexOperation::DocumentOperation { index_uid, .. }
+            IndexOperation::DocumentImport { index_uid, .. }
             | IndexOperation::DocumentDeletion { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
-            | IndexOperation::SettingsAndDocumentOperation { index_uid, .. } => index_uid,
+            | IndexOperation::SettingsAndDocumentImport { index_uid, .. } => index_uid,
         }
     }
 }
@@ -227,22 +222,17 @@ impl IndexScheduler {
                 },
                 must_create_index,
             })),
-            BatchKind::DocumentOperation { method, operation_ids, .. } => {
-                let tasks = self.get_existing_tasks(rtxn, operation_ids)?;
-                let primary_key = tasks
-                    .iter()
-                    .find_map(|task| match task.kind {
-                        KindWithContent::DocumentAdditionOrUpdate { ref primary_key, .. } => {
-                            // we want to stop on the first document addition
-                            Some(primary_key.clone())
-                        }
-                        KindWithContent::DocumentDeletion { .. } => None,
-                        _ => unreachable!(),
-                    })
-                    .flatten();
+            BatchKind::DocumentImport { method, import_ids, .. } => {
+                let tasks = self.get_existing_tasks(rtxn, import_ids)?;
+                let primary_key = match &tasks[0].kind {
+                    KindWithContent::DocumentAdditionOrUpdate { primary_key, .. } => {
+                        primary_key.clone()
+                    }
+                    _ => unreachable!(),
+                };
 
                 let mut documents_counts = Vec::new();
-                let mut operations = Vec::new();
+                let mut content_files = Vec::new();
 
                 for task in tasks.iter() {
                     match task.kind {
@@ -252,23 +242,19 @@ impl IndexScheduler {
                             ..
                         } => {
                             documents_counts.push(documents_count);
-                            operations.push(DocumentOperation::Add(content_file));
-                        }
-                        KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
-                            documents_counts.push(documents_ids.len() as u64);
-                            operations.push(DocumentOperation::Delete(documents_ids.clone()));
+                            content_files.push(content_file);
                         }
                         _ => unreachable!(),
                     }
                 }
 
                 Ok(Some(Batch::IndexOperation {
-                    op: IndexOperation::DocumentOperation {
+                    op: IndexOperation::DocumentImport {
                         index_uid,
                         primary_key,
                         method,
                         documents_counts,
-                        operations,
+                        content_files,
                         tasks,
                     },
                     must_create_index,
@@ -352,12 +338,12 @@ impl IndexScheduler {
                     must_create_index,
                 }))
             }
-            BatchKind::SettingsAndDocumentOperation {
+            BatchKind::SettingsAndDocumentImport {
                 settings_ids,
                 method,
                 allow_index_creation,
                 primary_key,
-                operation_ids,
+                import_ids,
             } => {
                 let settings = self.create_next_batch_index(
                     rtxn,
@@ -369,11 +355,11 @@ impl IndexScheduler {
                 let document_import = self.create_next_batch_index(
                     rtxn,
                     index_uid.clone(),
-                    BatchKind::DocumentOperation {
+                    BatchKind::DocumentImport {
                         method,
                         allow_index_creation,
                         primary_key,
-                        operation_ids,
+                        import_ids,
                     },
                     must_create_index,
                 )?;
@@ -382,10 +368,10 @@ impl IndexScheduler {
                     (
                         Some(Batch::IndexOperation {
                             op:
-                                IndexOperation::DocumentOperation {
+                                IndexOperation::DocumentImport {
                                     primary_key,
                                     documents_counts,
-                                    operations,
+                                    content_files,
                                     tasks: document_import_tasks,
                                     ..
                                 },
@@ -396,12 +382,12 @@ impl IndexScheduler {
                             ..
                         }),
                     ) => Ok(Some(Batch::IndexOperation {
-                        op: IndexOperation::SettingsAndDocumentOperation {
+                        op: IndexOperation::SettingsAndDocumentImport {
                             index_uid,
                             primary_key,
                             method,
                             documents_counts,
-                            operations,
+                            content_files,
                             document_import_tasks,
                             settings,
                             settings_tasks,
@@ -1017,12 +1003,12 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
-            IndexOperation::DocumentOperation {
+            IndexOperation::DocumentImport {
                 index_uid: _,
                 primary_key,
                 method,
-                documents_counts: _,
-                operations,
+                documents_counts,
+                content_files,
                 mut tasks,
             } => {
                 let mut primary_key_has_been_set = false;
@@ -1067,82 +1053,26 @@ impl IndexScheduler {
                     || must_stop_processing.get(),
                 )?;
 
-                for (operation, task) in operations.into_iter().zip(tasks.iter_mut()) {
-                    match operation {
-                        DocumentOperation::Add(content_uuid) => {
-                            let content_file = self.file_store.get_update(content_uuid)?;
-                            let reader = DocumentsBatchReader::from_reader(content_file)
-                                .map_err(milli::Error::from)?;
-                            let (new_builder, user_result) = builder.add_documents(reader)?;
-                            builder = new_builder;
+                let mut results = Vec::new();
+                for content_uuid in content_files.into_iter() {
+                    let content_file = self.file_store.get_update(content_uuid)?;
+                    let reader = DocumentsBatchReader::from_reader(content_file)
+                        .map_err(milli::Error::from)?;
+                    let (new_builder, user_result) = builder.add_documents(reader)?;
+                    builder = new_builder;
 
-                            let received_documents =
-                                if let Some(Details::DocumentAdditionOrUpdate {
-                                    received_documents,
-                                    ..
-                                }) = task.details
-                                {
-                                    received_documents
-                                } else {
-                                    // In the case of a `documentAdditionOrUpdate` the details MUST be set
-                                    unreachable!();
-                                };
+                    let user_result = match user_result {
+                        Ok(count) => Ok(DocumentAdditionResult {
+                            indexed_documents: count,
+                            number_of_documents: count, // TODO: this is wrong, we should use the value stored in the Details.
+                        }),
+                        Err(e) => Err(milli::Error::from(e)),
+                    };
 
-                            match user_result {
-                                Ok(count) => {
-                                    task.status = Status::Succeeded;
-                                    task.details = Some(Details::DocumentAdditionOrUpdate {
-                                        received_documents,
-                                        indexed_documents: Some(count),
-                                    })
-                                }
-                                Err(e) => {
-                                    task.status = Status::Failed;
-                                    task.details = Some(Details::DocumentAdditionOrUpdate {
-                                        received_documents,
-                                        indexed_documents: Some(0),
-                                    });
-                                    task.error = Some(milli::Error::from(e).into());
-                                }
-                            }
-                        }
-                        DocumentOperation::Delete(document_ids) => {
-                            let (new_builder, user_result) =
-                                builder.remove_documents(document_ids)?;
-                            builder = new_builder;
-
-                            let provided_ids =
-                                if let Some(Details::DocumentDeletion { provided_ids, .. }) =
-                                    task.details
-                                {
-                                    provided_ids
-                                } else {
-                                    // In the case of a `documentAdditionOrUpdate` the details MUST be set
-                                    unreachable!();
-                                };
-
-                            match user_result {
-                                Ok(count) => {
-                                    task.status = Status::Succeeded;
-                                    task.details = Some(Details::DocumentDeletion {
-                                        provided_ids,
-                                        deleted_documents: Some(count),
-                                    });
-                                }
-                                Err(e) => {
-                                    task.status = Status::Failed;
-                                    task.details = Some(Details::DocumentDeletion {
-                                        provided_ids,
-                                        deleted_documents: Some(0),
-                                    });
-                                    task.error = Some(milli::Error::from(e).into());
-                                }
-                            }
-                        }
-                    }
+                    results.push(user_result);
                 }
 
-                if !tasks.iter().all(|res| res.error.is_some()) {
+                if results.iter().any(|res| res.is_ok()) {
                     let addition = builder.execute()?;
                     info!("document addition done: {:?}", addition);
                 } else if primary_key_has_been_set {
@@ -1157,6 +1087,29 @@ impl IndexScheduler {
                     )?;
                 }
 
+                for (task, (ret, count)) in
+                    tasks.iter_mut().zip(results.into_iter().zip(documents_counts))
+                {
+                    match ret {
+                        Ok(DocumentAdditionResult { indexed_documents, number_of_documents }) => {
+                            task.status = Status::Succeeded;
+                            task.details = Some(Details::DocumentAdditionOrUpdate {
+                                received_documents: number_of_documents,
+                                indexed_documents: Some(indexed_documents),
+                            });
+                        }
+                        Err(error) => {
+                            task.status = Status::Failed;
+                            task.details = Some(Details::DocumentAdditionOrUpdate {
+                                received_documents: count,
+                                // if there was an error we indexed 0 documents.
+                                indexed_documents: Some(0),
+                            });
+                            task.error = Some(error.into())
+                        }
+                    }
+                }
+
                 Ok(tasks)
             }
             IndexOperation::DocumentDeletion { index_uid: _, documents, mut tasks } => {
@@ -1199,12 +1152,12 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
-            IndexOperation::SettingsAndDocumentOperation {
+            IndexOperation::SettingsAndDocumentImport {
                 index_uid,
                 primary_key,
                 method,
                 documents_counts,
-                operations,
+                content_files,
                 document_import_tasks,
                 settings,
                 settings_tasks,
@@ -1222,12 +1175,12 @@ impl IndexScheduler {
                 let mut import_tasks = self.apply_index_operation(
                     index_wtxn,
                     index,
-                    IndexOperation::DocumentOperation {
+                    IndexOperation::DocumentImport {
                         index_uid,
                         primary_key,
                         method,
                         documents_counts,
-                        operations,
+                        content_files,
                         tasks: document_import_tasks,
                     },
                 )?;

index-scheduler/src/index_mapper.rs

@@ -1,21 +1,20 @@
-use std::collections::hash_map::Entry;
-use std::collections::HashMap;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 use std::{fs, thread};
 
 use log::error;
 use meilisearch_types::heed::types::Str;
-use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn};
+use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::Index;
-use synchronoise::SignalEvent;
 use time::OffsetDateTime;
 use uuid::Uuid;
 
-use self::IndexStatus::{Available, BeingDeleted, BeingResized};
+use self::index_map::IndexMap;
+use self::IndexStatus::{Available, BeingDeleted, Closing, Missing};
 use crate::uuid_codec::UuidCodec;
-use crate::{clamp_to_page_size, Error, Result};
+use crate::{Error, Result};
 
 const INDEX_MAPPING: &str = "index-mapping";
 
@@ -26,52 +25,323 @@ const INDEX_MAPPING: &str = "index-mapping";
 /// 2. Opening indexes and storing references to these opened indexes
 /// 3. Accessing indexes through their uuid
 /// 4. Mapping a user-defined name to each index uuid.
+///
+/// # Implementation notes
+///
+/// An index exists as 3 bits of data:
+/// 1. The index data on disk, that can exist in 3 states: Missing, Present, or BeingDeleted.
+/// 2. The persistent database containing the association between the index' name and its UUID,
+///    that can exist in 2 states: Missing or Present.
+/// 3. The state of the index in the in-memory `IndexMap`, that can exist in multiple states:
+///   - Missing
+///   - Available
+///   - Closing (because an index needs resizing or was evicted from the cache)
+///   - BeingDeleted
+///
+/// All of this data should be kept consistent between index operations, which is achieved by the `IndexMapper`
+/// with the use of the following primitives:
+/// - A RwLock on the `IndexMap`.
+/// - Transactions on the association database.
+/// - ClosingEvent signals emitted when closing an environment.
 #[derive(Clone)]
 pub struct IndexMapper {
     /// Keep track of the opened indexes. Used mainly by the index resolver.
-    index_map: Arc<RwLock<HashMap<Uuid, IndexStatus>>>,
+    index_map: Arc<RwLock<IndexMap>>,
 
     /// Map an index name with an index uuid currently available on disk.
     pub(crate) index_mapping: Database<Str, UuidCodec>,
 
     /// Path to the folder where the LMDB environments of each index are.
     base_path: PathBuf,
-    index_size: usize,
+    /// The map size an index is opened with on the first time.
+    index_base_map_size: usize,
+    /// The quantity by which the map size of an index is incremented upon reopening, in bytes.
+    index_growth_amount: usize,
     pub indexer_config: Arc<IndexerConfig>,
 }
 
-/// Whether the index is available for use or is forbidden to be inserted back in the index map
-#[allow(clippy::large_enum_variant)]
-#[derive(Clone)]
-pub enum IndexStatus {
-    /// Do not insert it back in the index map as it is currently being deleted.
-    BeingDeleted,
-    /// Temporarily do not insert the index in the index map as it is currently being resized.
-    BeingResized(Arc<SignalEvent>),
-    /// You can use the index without worrying about anything.
-    Available(Index),
-}
+mod index_map {
+    /// the map size to use when we don't succeed in reading it in indexes.
+    const DEFAULT_MAP_SIZE: usize = 10 * 1024 * 1024 * 1024; // 10 GiB
 
-impl IndexMapper {
-    pub fn new(
-        env: &Env,
-        base_path: PathBuf,
-        index_size: usize,
-        indexer_config: IndexerConfig,
-    ) -> Result<Self> {
-        Ok(Self {
-            index_map: Arc::default(),
-            index_mapping: env.create_database(Some(INDEX_MAPPING))?,
-            base_path,
-            index_size,
-            indexer_config: Arc::new(indexer_config),
-        })
+    use std::collections::BTreeMap;
+    use std::path::Path;
+    use std::time::Duration;
+
+    use meilisearch_types::heed::{EnvClosingEvent, EnvOpenOptions};
+    use meilisearch_types::milli::Index;
+    use time::OffsetDateTime;
+    use uuid::Uuid;
+
+    use super::IndexStatus::{self, Available, BeingDeleted, Closing, Missing};
+    use crate::lru::{InsertionOutcome, LruMap};
+    use crate::{clamp_to_page_size, Result};
+
+    /// Keep an internally consistent view of the open indexes in memory.
+    ///
+    /// This view is made of an LRU cache that will evict the least frequently used indexes when new indexes are opened.
+    /// Indexes that are being closed (for resizing or due to cache eviction) or deleted cannot be evicted from the cache and
+    /// are stored separately.
+    ///
+    /// This view provides operations to change the state of the index as it is known in memory:
+    /// open an index (making it available for queries), close an index (specifying the new size it should be opened with),
+    /// delete an index.
+    ///
+    /// External consistency with the other bits of data of an index is provided by the `IndexMapper` parent structure.
+    pub struct IndexMap {
+        /// A LRU map of indexes that are in the open state and available for queries.
+        available: LruMap<Uuid, Index>,
+        /// A map of indexes that are not available for queries, either because they are being deleted
+        /// or because they are being closed.
+        ///
+        /// If they are being deleted, the UUID points to `None`.
+        unavailable: BTreeMap<Uuid, Option<ClosingIndex>>,
+
+        /// A monotonically increasing generation number, used to differentiate between multiple successive index closing requests.
+        ///
+        /// Because multiple readers could be waiting on an index to close, the following could theoretically happen:
+        ///
+        /// 1. Multiple readers wait for the index closing to occur.
+        /// 2. One of them "wins the race", takes the lock and then removes the index that finished closing from the map.
+        /// 3. The index is reopened, but must be closed again (such as being resized again).
+        /// 4. One reader that "lost the race" in (2) wakes up and tries to take the lock and remove the index from the map.
+        ///
+        /// In that situation, the index may or may not have finished closing. The `generation` field allows to remember which
+        /// closing request was made, so the reader that "lost the race" has the old generation and will need to wait again for the index
+        /// to close.
+        generation: usize,
+    }
+
+    #[derive(Clone)]
+    pub struct ClosingIndex {
+        uuid: Uuid,
+        closing_event: EnvClosingEvent,
+        map_size: usize,
+        generation: usize,
+    }
+
+    impl ClosingIndex {
+        /// Waits for the index to be definitely closed.
+        ///
+        /// To avoid blocking, users should relinquish their locks to the IndexMap before calling this function.
+        ///
+        /// After the index is physically closed, the in memory map must still be updated to take this into account.
+        /// To do so, a `ReopenableIndex` is returned, that can be used to either definitely close or definitely open
+        /// the index without waiting anymore.
+        pub fn wait_timeout(self, timeout: Duration) -> Option<ReopenableIndex> {
+            self.closing_event.wait_timeout(timeout).then_some(ReopenableIndex {
+                uuid: self.uuid,
+                map_size: self.map_size,
+                generation: self.generation,
+            })
+        }
+    }
+
+    pub struct ReopenableIndex {
+        uuid: Uuid,
+        map_size: usize,
+        generation: usize,
+    }
+
+    impl ReopenableIndex {
+        /// Attempts to reopen the index, which can result in the index being reopened again or not
+        /// (e.g. if another thread already opened and closed the index again).
+        ///
+        /// Use get again on the IndexMap to get the updated status.
+        ///
+        /// Fails if the underlying index creation fails.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status |
+        /// |-----------------|------------|
+        /// | Missing | Missing |
+        /// | BeingDeleted | BeingDeleted |
+        /// | Closing | Available or Closing depending on generation |
+        /// | Available | Available |
+        ///
+        pub fn reopen(self, map: &mut IndexMap, path: &Path) -> Result<()> {
+            if let Closing(reopen) = map.get(&self.uuid) {
+                if reopen.generation != self.generation {
+                    return Ok(());
+                }
+                map.unavailable.remove(&self.uuid);
+                map.create(&self.uuid, path, None, self.map_size)?;
+            }
+            Ok(())
+        }
+
+        /// Attempts to close the index, which may or may not result in the index being closed
+        /// (e.g. if another thread already reopened the index again).
+        ///
+        /// Use get again on the IndexMap to get the updated status.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status |
+        /// |-----------------|------------|
+        /// | Missing | Missing |
+        /// | BeingDeleted | BeingDeleted |
+        /// | Closing | Missing or Closing depending on generation |
+        /// | Available | Available |
+        pub fn close(self, map: &mut IndexMap) {
+            if let Closing(reopen) = map.get(&self.uuid) {
+                if reopen.generation != self.generation {
+                    return;
+                }
+                map.unavailable.remove(&self.uuid);
+            }
+        }
+    }
+
+    impl IndexMap {
+        pub fn new(cap: usize) -> IndexMap {
+            Self { unavailable: Default::default(), available: LruMap::new(cap), generation: 0 }
+        }
+
+        /// Gets the current status of an index in the map.
+        ///
+        /// If the index is available it can be accessed from the returned status.
+        pub fn get(&self, uuid: &Uuid) -> IndexStatus {
+            self.available
+                .get(uuid)
+                .map(|index| Available(index.clone()))
+                .unwrap_or_else(|| self.get_unavailable(uuid))
+        }
+
+        fn get_unavailable(&self, uuid: &Uuid) -> IndexStatus {
+            match self.unavailable.get(uuid) {
+                Some(Some(reopen)) => Closing(reopen.clone()),
+                Some(None) => BeingDeleted,
+                None => Missing,
+            }
+        }
+
+        /// Attempts to create a new index that wasn't existing before.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status |
+        /// |-----------------|------------|
+        /// | Missing | Available |
+        /// | BeingDeleted | panics |
+        /// | Closing | panics |
+        /// | Available | panics |
+        ///
+        pub fn create(
+            &mut self,
+            uuid: &Uuid,
+            path: &Path,
+            date: Option<(OffsetDateTime, OffsetDateTime)>,
+            map_size: usize,
+        ) -> Result<Index> {
+            if !matches!(self.get_unavailable(uuid), Missing) {
+                panic!("Attempt to open an index that was unavailable");
+            }
+            let index = create_or_open_index(path, date, map_size)?;
+            match self.available.insert(*uuid, index.clone()) {
+                InsertionOutcome::InsertedNew => (),
+                InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
+                    self.close(evicted_uuid, evicted_index, 0);
+                }
+                InsertionOutcome::Replaced(_) => {
+                    panic!("Attempt to open an index that was already opened")
+                }
+            }
+            Ok(index)
+        }
+
+        /// Increases the current generation. See documentation for this field.
+        ///
+        /// In the unlikely event that the 2^64 generations would have been exhausted, we simply wrap-around.
+        ///
+        /// For this to cause an issue, one should be able to stop a reader in time after it got a `ReopenableIndex` and before it takes the lock
+        /// to remove it from the unavailable map, and keep the reader in this frozen state for 2^64 closing of other indexes.
+        ///
+        /// This seems overwhelmingly impossible to achieve in practice.
+        fn next_generation(&mut self) -> usize {
+            self.generation = self.generation.wrapping_add(1);
+            self.generation
+        }
+
+        /// Attempts to close an index.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status |
+        /// |-----------------|------------|
+        /// | Missing | Missing |
+        /// | BeingDeleted | BeingDeleted |
+        /// | Closing | Closing |
+        /// | Available | Closing |
+        ///
+        pub fn close_for_resize(&mut self, uuid: &Uuid, map_size_growth: usize) {
+            let Some(index) = self.available.remove(uuid) else { return; };
+            self.close(*uuid, index, map_size_growth);
+        }
+
+        fn close(&mut self, uuid: Uuid, index: Index, map_size_growth: usize) {
+            let map_size = index.map_size().unwrap_or(DEFAULT_MAP_SIZE) + map_size_growth;
+            let closing_event = index.prepare_for_closing();
+            let generation = self.next_generation();
+            self.unavailable
+                .insert(uuid, Some(ClosingIndex { uuid, closing_event, map_size, generation }));
+        }
+
+        /// Attempts to delete and index.
+        ///
+        ///  `end_deletion` must be called just after.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status | Return value |
+        /// |-----------------|------------|--------------|
+        /// | Missing | BeingDeleted | Ok(None) |
+        /// | BeingDeleted | BeingDeleted | Err(None) |
+        /// | Closing | Closing | Err(Some(reopen)) |
+        /// | Available | BeingDeleted | Ok(Some(env_closing_event)) |
+        pub fn start_deletion(
+            &mut self,
+            uuid: &Uuid,
+        ) -> std::result::Result<Option<EnvClosingEvent>, Option<ClosingIndex>> {
+            if let Some(index) = self.available.remove(uuid) {
+                return Ok(Some(index.prepare_for_closing()));
+            }
+            match self.unavailable.remove(uuid) {
+                Some(Some(reopen)) => Err(Some(reopen)),
+                Some(None) => Err(None),
+                None => Ok(None),
+            }
+        }
+
+        /// Marks that an index deletion finished.
+        ///
+        /// Must be used after calling `start_deletion`.
+        ///
+        /// # Status table
+        ///
+        /// | Previous Status | New Status |
+        /// |-----------------|------------|
+        /// | Missing | Missing |
+        /// | BeingDeleted | Missing |
+        /// | Closing | panics |
+        /// | Available | panics |
+        pub fn end_deletion(&mut self, uuid: &Uuid) {
+            assert!(
+                self.available.get(uuid).is_none(),
+                "Attempt to finish deletion of an index that was not being deleted"
+            );
+            // Do not panic if the index was Missing or BeingDeleted
+            assert!(
+                !matches!(self.unavailable.remove(uuid), Some(Some(_))),
+                "Attempt to finish deletion of an index that was being closed"
+            );
+        }
     }
 
     /// Create or open an index in the specified path.
-    /// The path *must* exists or an error will be thrown.
+    /// The path *must* exist or an error will be thrown.
     fn create_or_open_index(
-        &self,
         path: &Path,
         date: Option<(OffsetDateTime, OffsetDateTime)>,
         map_size: usize,
@@ -87,6 +357,114 @@ impl IndexMapper {
         }
     }
 
+    /// Putting the tests of the LRU down there so we have access to the cache's private members
+    #[cfg(test)]
+    mod tests {
+
+        use meilisearch_types::heed::Env;
+        use meilisearch_types::Index;
+        use uuid::Uuid;
+
+        use super::super::IndexMapper;
+        use crate::tests::IndexSchedulerHandle;
+        use crate::utils::clamp_to_page_size;
+        use crate::IndexScheduler;
+
+        impl IndexMapper {
+            fn test() -> (Self, Env, IndexSchedulerHandle) {
+                let (index_scheduler, handle) = IndexScheduler::test(true, vec![]);
+                (index_scheduler.index_mapper, index_scheduler.env, handle)
+            }
+        }
+
+        fn check_first_unavailable(mapper: &IndexMapper, expected_uuid: Uuid, is_closing: bool) {
+            let index_map = mapper.index_map.read().unwrap();
+            let (uuid, state) = index_map.unavailable.first_key_value().unwrap();
+            assert_eq!(uuid, &expected_uuid);
+            assert_eq!(state.is_some(), is_closing);
+        }
+
+        #[test]
+        fn evict_indexes() {
+            let (mapper, env, _handle) = IndexMapper::test();
+            let mut uuids = vec![];
+            // LRU cap + 1
+            for i in 0..(5 + 1) {
+                let index_name = format!("index-{i}");
+                let wtxn = env.write_txn().unwrap();
+                mapper.create_index(wtxn, &index_name, None).unwrap();
+                let txn = env.read_txn().unwrap();
+                uuids.push(mapper.index_mapping.get(&txn, &index_name).unwrap().unwrap());
+            }
+            // index-0 was evicted
+            check_first_unavailable(&mapper, uuids[0], true);
+
+            // get back the evicted index
+            let wtxn = env.write_txn().unwrap();
+            mapper.create_index(wtxn, "index-0", None).unwrap();
+
+            // Least recently used is now index-1
+            check_first_unavailable(&mapper, uuids[1], true);
+        }
+
+        #[test]
+        fn resize_index() {
+            let (mapper, env, _handle) = IndexMapper::test();
+            let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
+            assert_index_size(index, mapper.index_base_map_size);
+
+            mapper.resize_index(&env.read_txn().unwrap(), "index").unwrap();
+
+            let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
+            assert_index_size(index, mapper.index_base_map_size + mapper.index_growth_amount);
+
+            mapper.resize_index(&env.read_txn().unwrap(), "index").unwrap();
+
+            let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
+            assert_index_size(index, mapper.index_base_map_size + mapper.index_growth_amount * 2);
+        }
+
+        fn assert_index_size(index: Index, expected: usize) {
+            let expected = clamp_to_page_size(expected);
+            let index_map_size = index.map_size().unwrap();
+            assert_eq!(index_map_size, expected);
+        }
+    }
+}
+
+/// Whether the index is available for use or is forbidden to be inserted back in the index map
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone)]
+pub enum IndexStatus {
+    /// Not currently in the index map.
+    Missing,
+    /// Do not insert it back in the index map as it is currently being deleted.
+    BeingDeleted,
+    /// Temporarily do not insert the index in the index map as it is currently being resized/evicted from the map.
+    Closing(index_map::ClosingIndex),
+    /// You can use the index without worrying about anything.
+    Available(Index),
+}
+
+impl IndexMapper {
+    pub fn new(
+        env: &Env,
+        base_path: PathBuf,
+        index_base_map_size: usize,
+        index_growth_amount: usize,
+        index_count: usize,
+        indexer_config: IndexerConfig,
+    ) -> Result<Self> {
+        Ok(Self {
+            index_map: Arc::new(RwLock::new(IndexMap::new(index_count))),
+            index_mapping: env.create_database(Some(INDEX_MAPPING))?,
+            base_path,
+            index_base_map_size,
+            index_growth_amount,
+            indexer_config: Arc::new(indexer_config),
+        })
+    }
+
     /// Get or create the index.
     pub fn create_index(
         &self,
@@ -106,16 +484,17 @@ impl IndexMapper {
                 let index_path = self.base_path.join(uuid.to_string());
                 fs::create_dir_all(&index_path)?;
 
-                let index = self.create_or_open_index(&index_path, date, self.index_size)?;
-
-                wtxn.commit()?;
                 // Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
                 // This is very unlikely to happen in practice.
                 // TODO: it would be better to lazily create the index. But we need an Index::open function for milli.
-                if self.index_map.write().unwrap().insert(uuid, Available(index.clone())).is_some()
-                {
-                    panic!("Uuid v4 conflict: index with UUID {uuid} already exists.");
-                }
+                let index = self.index_map.write().unwrap().create(
+                    &uuid,
+                    &index_path,
+                    date,
+                    self.index_base_map_size,
+                )?;
+
+                wtxn.commit()?;
 
                 Ok(index)
             }
@@ -135,23 +514,30 @@ impl IndexMapper {
         assert!(self.index_mapping.delete(&mut wtxn, name)?);
 
         wtxn.commit()?;
+
+        let mut tries = 0;
         // We remove the index from the in-memory index map.
         let closing_event = loop {
             let mut lock = self.index_map.write().unwrap();
-            let resize_operation = match lock.insert(uuid, BeingDeleted) {
-                Some(Available(index)) => break Some(index.prepare_for_closing()),
-                // The target index is in the middle of a resize operation.
-                // Wait for this operation to complete, then try again.
-                Some(BeingResized(resize_operation)) => resize_operation.clone(),
-                // The index is already being deleted or doesn't exist.
-                // It's OK to remove it from the map again.
-                _ => break None,
-            };
-
-            // Avoiding deadlocks: we need to drop the lock before waiting for the end of the resize, which
-            // will involve operations on the very map we're locking.
-            drop(lock);
-            resize_operation.wait();
+            match lock.start_deletion(&uuid) {
+                Ok(env_closing) => break env_closing,
+                Err(Some(reopen)) => {
+                    // drop the lock here so that we don't synchronously wait for the index to close.
+                    drop(lock);
+                    tries += 1;
+                    if tries >= 100 {
+                        panic!("Too many attempts to close index {name} prior to deletion.")
+                    }
+                    let reopen = if let Some(reopen) = reopen.wait_timeout(Duration::from_secs(6)) {
+                        reopen
+                    } else {
+                        continue;
+                    };
+                    reopen.close(&mut self.index_map.write().unwrap());
+                    continue;
+                }
+                Err(None) => return Ok(()),
+            }
         };
 
         let index_map = self.index_map.clone();
@@ -161,7 +547,7 @@ impl IndexMapper {
             .name(String::from("index_deleter"))
             .spawn(move || {
                 // We first wait to be sure that the previously opened index is effectively closed.
-                // This can take a lot of time, this is why we do that in a seperate thread.
+                // This can take a lot of time, this is why we do that in a separate thread.
                 if let Some(closing_event) = closing_event {
                     closing_event.wait();
                 }
@@ -175,7 +561,7 @@ impl IndexMapper {
                 }
 
                 // Finally we remove the entry from the index map.
-                assert!(matches!(index_map.write().unwrap().remove(&uuid), Some(BeingDeleted)));
+                index_map.write().unwrap().end_deletion(&uuid);
             })
             .unwrap();
 
@@ -195,76 +581,15 @@ impl IndexMapper {
     /// - If the Index corresponding to the passed name is concurrently being deleted/resized or cannot be found in the
     ///   in memory hash map.
     pub fn resize_index(&self, rtxn: &RoTxn, name: &str) -> Result<()> {
-        // fixme: factor to a function?
         let uuid = self
             .index_mapping
             .get(rtxn, name)?
             .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
 
         // We remove the index from the in-memory index map.
-        let mut lock = self.index_map.write().unwrap();
-        // signal that will be sent when the resize operation completes
-        let resize_operation = Arc::new(SignalEvent::manual(false));
-        let index = match lock.insert(uuid, BeingResized(resize_operation)) {
-            Some(Available(index)) => index,
-            Some(previous_status) => {
-                lock.insert(uuid, previous_status);
-                panic!(
-                    "Attempting to resize index {name} that is already being resized or deleted."
-                )
-            }
-            None => {
-                panic!("Could not find the status of index {name} in the in-memory index mapper.")
-            }
-        };
+        self.index_map.write().unwrap().close_for_resize(&uuid, self.index_growth_amount);
 
-        drop(lock);
-
-        let resize_succeeded = (move || {
-            let current_size = index.map_size()?;
-            let new_size = current_size * 2;
-            let closing_event = index.prepare_for_closing();
-
-            log::debug!("Waiting for index {name} to close");
-
-            if !closing_event.wait_timeout(std::time::Duration::from_secs(600)) {
-                // fail after 10 minutes waiting
-                panic!("Could not resize index {name} (unable to close it)");
-            }
-
-            log::info!("Resized index {name} from {current_size} to {new_size} bytes");
-            let index_path = self.base_path.join(uuid.to_string());
-            let index = self.create_or_open_index(&index_path, None, new_size)?;
-            Ok(index)
-        })();
-
-        // Put the map back to a consistent state.
-        // Even if there was an error we don't want to leave the map in an inconsistent state as it would cause
-        // deadlocks.
-        let mut lock = self.index_map.write().unwrap();
-        let (resize_operation, resize_succeeded) = match resize_succeeded {
-            Ok(index) => {
-                // insert the resized index
-                let Some(BeingResized(resize_operation)) = lock.insert(uuid, Available(index)) else {
-                    panic!("Index state for index {name} was modified while it was being resized")
-                };
-
-                (resize_operation, Ok(()))
-            }
-            Err(error) => {
-                // there was an error, not much we can do... delete the index from the in-memory map to prevent future errors
-                let Some(BeingResized(resize_operation)) = lock.remove(&uuid) else {
-                                    panic!("Index state for index {name} was modified while it was being resized")
-                                };
-                (resize_operation, Err(error))
-            }
-        };
-
-        // drop the lock before signaling completion so that other threads don't immediately await on the lock after waking up.
-        drop(lock);
-        resize_operation.signal();
-
-        resize_succeeded
+        Ok(())
     }
 
     /// Return an index, may open it if it wasn't already opened.
@@ -275,46 +600,53 @@ impl IndexMapper {
             .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
 
         // we clone here to drop the lock before entering the match
+        let mut tries = 0;
         let index = loop {
-            let index = self.index_map.read().unwrap().get(&uuid).cloned();
+            tries += 1;
+            if tries > 100 {
+                panic!("Too many spurious wake ups while the index is being resized");
+            }
+            let index = self.index_map.read().unwrap().get(&uuid);
 
             match index {
-                Some(Available(index)) => break index,
-                Some(BeingResized(ref resize_operation)) => {
+                Available(index) => break index,
+                Closing(reopen) => {
                     // Avoiding deadlocks: no lock taken while doing this operation.
-                    resize_operation.wait();
+                    let reopen = if let Some(reopen) = reopen.wait_timeout(Duration::from_secs(6)) {
+                        reopen
+                    } else {
+                        continue;
+                    };
+                    let index_path = self.base_path.join(uuid.to_string());
+                    // take the lock to reopen the environment.
+                    reopen.reopen(&mut self.index_map.write().unwrap(), &index_path)?;
                     continue;
                 }
-                Some(BeingDeleted) => return Err(Error::IndexNotFound(name.to_string())),
+                BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
                 // since we're lazy, it's possible that the index has not been opened yet.
-                None => {
+                Missing => {
                     let mut index_map = self.index_map.write().unwrap();
                     // between the read lock and the write lock it's not impossible
-                    // that someone already opened the index (eg if two search happens
+                    // that someone already opened the index (eg if two searches happen
                     // at the same time), thus before opening it we check a second time
                     // if it's not already there.
-                    // Since there is a good chance it's not already there we can use
-                    // the entry method.
-                    match index_map.entry(uuid) {
-                        Entry::Vacant(entry) => {
+                    match index_map.get(&uuid) {
+                        Missing => {
                             let index_path = self.base_path.join(uuid.to_string());
 
-                            let index =
-                                self.create_or_open_index(&index_path, None, self.index_size)?;
-                            entry.insert(Available(index.clone()));
-                            break index;
+                            break index_map.create(
+                                &uuid,
+                                &index_path,
+                                None,
+                                self.index_base_map_size,
+                            )?;
                         }
-                        Entry::Occupied(entry) => match entry.get() {
-                            Available(index) => break index.clone(),
-                            BeingResized(resize_operation) => {
-                                // Avoiding the deadlock: we drop the lock before waiting
-                                let resize_operation = resize_operation.clone();
-                                drop(index_map);
-                                resize_operation.wait();
-                                continue;
-                            }
-                            BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
-                        },
+                        Available(index) => break index,
+                        Closing(_) => {
+                            // the reopening will be handled in the next loop operation
+                            continue;
+                        }
+                        BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
                     }
                 }
             }

index-scheduler/src/lib.rs

@@ -24,6 +24,7 @@ pub mod error;
 mod index_mapper;
 #[cfg(test)]
 mod insta_snapshot;
+mod lru;
 mod utils;
 mod uuid_codec;
 
@@ -31,7 +32,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub type TaskId = u32;
 
 use std::ops::{Bound, RangeBounds};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
 use std::sync::atomic::Ordering::Relaxed;
 use std::sync::{Arc, RwLock};
@@ -230,8 +231,12 @@ pub struct IndexSchedulerOptions {
     pub dumps_path: PathBuf,
     /// The maximum size, in bytes, of the task index.
     pub task_db_size: usize,
-    /// The maximum size, in bytes, of each meilisearch index.
-    pub index_size: usize,
+    /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index.
+    pub index_base_map_size: usize,
+    /// The size, in bytes, by which the map size of an index is increased when it resized due to being full.
+    pub index_growth_amount: usize,
+    /// The number of indexes that can be concurrently opened in memory.
+    pub index_count: usize,
     /// Configuration used during indexing for each meilisearch index.
     pub indexer_config: IndexerConfig,
     /// Set to `true` iff the index scheduler is allowed to automatically
@@ -361,9 +366,25 @@ impl IndexScheduler {
         std::fs::create_dir_all(&options.indexes_path)?;
         std::fs::create_dir_all(&options.dumps_path)?;
 
+        let task_db_size = clamp_to_page_size(options.task_db_size);
+        let budget = if options.indexer_config.skip_index_budget {
+            IndexBudget {
+                map_size: options.index_base_map_size,
+                index_count: options.index_count,
+                task_db_size,
+            }
+        } else {
+            Self::index_budget(
+                &options.tasks_path,
+                options.index_base_map_size,
+                task_db_size,
+                options.index_count,
+            )
+        };
+
         let env = heed::EnvOpenOptions::new()
             .max_dbs(10)
-            .map_size(clamp_to_page_size(options.task_db_size))
+            .map_size(budget.task_db_size)
             .open(options.tasks_path)?;
         let file_store = FileStore::new(&options.update_file_path)?;
 
@@ -383,7 +404,9 @@ impl IndexScheduler {
             index_mapper: IndexMapper::new(
                 &env,
                 options.indexes_path,
-                options.index_size,
+                budget.map_size,
+                options.index_growth_amount,
+                budget.index_count,
                 options.indexer_config,
             )?,
             env,
@@ -407,6 +430,60 @@ impl IndexScheduler {
         Ok(this)
     }
 
+    fn index_budget(
+        tasks_path: &Path,
+        base_map_size: usize,
+        mut task_db_size: usize,
+        max_index_count: usize,
+    ) -> IndexBudget {
+        let budget = utils::dichotomic_search(base_map_size, |map_size| {
+            Self::is_good_heed(tasks_path, map_size)
+        });
+
+        log::info!("memmap budget: {budget}B");
+        let mut budget = budget / 2;
+        if task_db_size > (budget / 2) {
+            task_db_size = clamp_to_page_size(budget * 2 / 5);
+            log::warn!(
+                "Decreasing max size of task DB to {task_db_size}B due to constrained memory space"
+            );
+        }
+        budget -= task_db_size;
+
+        // won't be mutated again
+        let budget = budget;
+        let task_db_size = task_db_size;
+
+        log::info!("index budget: {budget}B");
+        let mut index_count = budget / base_map_size;
+        if index_count < 2 {
+            // take a bit less than half than the budget to make sure we can always afford to open an index
+            let map_size = (budget * 2) / 5;
+            // single index of max budget
+            log::warn!("1 index of {map_size}B can be opened simultaneously.");
+            return IndexBudget { map_size, index_count: 1, task_db_size };
+        }
+        // give us some space for an additional index when the cache is already full
+        // decrement is OK because index_count >= 2.
+        index_count -= 1;
+        if index_count > max_index_count {
+            index_count = max_index_count;
+        }
+        log::info!("Up to {index_count} indexes of {base_map_size}B opened simultaneously.");
+        IndexBudget { map_size: base_map_size, index_count, task_db_size }
+    }
+
+    fn is_good_heed(tasks_path: &Path, map_size: usize) -> bool {
+        if let Ok(env) =
+            heed::EnvOpenOptions::new().map_size(clamp_to_page_size(map_size)).open(tasks_path)
+        {
+            env.prepare_for_closing().wait();
+            true
+        } else {
+            false
+        }
+    }
+
     pub fn read_txn(&self) -> Result<RoTxn> {
         self.env.read_txn().map_err(|e| e.into())
     }
@@ -1111,6 +1188,16 @@ pub enum TickOutcome {
     WaitForSignal,
 }
 
+/// How many indexes we can afford to have open simultaneously.
+struct IndexBudget {
+    /// Map size of an index.
+    map_size: usize,
+    /// Index count of an index.
+    index_count: usize,
+    /// For very constrained systems we might need to reduce the base task_db_size so we can accept at least one index.
+    task_db_size: usize,
+}
+
 #[cfg(test)]
 mod tests {
     use std::io::{BufWriter, Seek, Write};
@@ -1154,6 +1241,8 @@ mod tests {
             let tempdir = TempDir::new().unwrap();
             let (sender, receiver) = crossbeam::channel::bounded(0);
 
+            let indexer_config = IndexerConfig { skip_index_budget: true, ..Default::default() };
+
             let options = IndexSchedulerOptions {
                 version_file_path: tempdir.path().join(VERSION_FILE_NAME),
                 auth_path: tempdir.path().join("auth"),
@@ -1163,8 +1252,10 @@ mod tests {
                 snapshots_path: tempdir.path().join("snapshots"),
                 dumps_path: tempdir.path().join("dumps"),
                 task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
-                index_size: 1000 * 1000,   // 1 MB, we don't use MiB on purpose.
-                indexer_config: IndexerConfig::default(),
+                index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
+                index_growth_amount: 1000 * 1000, // 1 MB
+                index_count: 5,
+                indexer_config,
                 autobatching_enabled,
             };
 
@@ -1706,105 +1797,6 @@ mod tests {
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "both_task_succeeded");
     }
 
-    #[test]
-    fn document_addition_and_document_deletion() {
-        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
-
-        let content = r#"[
-            { "id": 1, "doggo": "jean bob" },
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "doggo": "bork" }
-        ]"#;
-
-        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
-        let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap();
-        file.persist().unwrap();
-        index_scheduler
-            .register(KindWithContent::DocumentAdditionOrUpdate {
-                index_uid: S("doggos"),
-                primary_key: Some(S("id")),
-                method: ReplaceDocuments,
-                content_file: uuid,
-                documents_count,
-                allow_index_creation: true,
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task");
-        index_scheduler
-            .register(KindWithContent::DocumentDeletion {
-                index_uid: S("doggos"),
-                documents_ids: vec![S("1"), S("2")],
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task");
-
-        handle.advance_one_successful_batch(); // The addition AND deletion should've been batched together
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_the_batch");
-
-        let index = index_scheduler.index("doggos").unwrap();
-        let rtxn = index.read_txn().unwrap();
-        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
-        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
-        let documents = index
-            .all_documents(&rtxn)
-            .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
-            .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
-    }
-
-    #[test]
-    fn document_deletion_and_document_addition() {
-        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
-        index_scheduler
-            .register(KindWithContent::DocumentDeletion {
-                index_uid: S("doggos"),
-                documents_ids: vec![S("1"), S("2")],
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_first_task");
-
-        let content = r#"[
-            { "id": 1, "doggo": "jean bob" },
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "doggo": "bork" }
-        ]"#;
-
-        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
-        let documents_count = read_json(content.as_bytes(), file.as_file_mut()).unwrap();
-        file.persist().unwrap();
-        index_scheduler
-            .register(KindWithContent::DocumentAdditionOrUpdate {
-                index_uid: S("doggos"),
-                primary_key: Some(S("id")),
-                method: ReplaceDocuments,
-                content_file: uuid,
-                documents_count,
-                allow_index_creation: true,
-            })
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_second_task");
-
-        // The deletion should have failed because it can't create an index
-        handle.advance_one_failed_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_failing_the_deletion");
-
-        // The addition should works
-        handle.advance_one_successful_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_last_successful_addition");
-
-        let index = index_scheduler.index("doggos").unwrap();
-        let rtxn = index.read_txn().unwrap();
-        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
-        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
-        let documents = index
-            .all_documents(&rtxn)
-            .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
-            .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
-    }
-
     #[test]
     fn do_not_batch_task_of_different_indexes() {
         let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);

index-scheduler/src/lru.rs

@@ -0,0 +1,202 @@
+//! Thread-safe `Vec`-backend LRU cache using [`std::sync::atomic::AtomicU64`] for synchronization.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Thread-safe `Vec`-backend LRU cache
+#[derive(Debug)]
+pub struct Lru<T> {
+    data: Vec<(AtomicU64, T)>,
+    generation: AtomicU64,
+    cap: usize,
+}
+
+impl<T> Lru<T> {
+    /// Creates a new LRU cache with the specified capacity.
+    ///
+    /// The capacity is allocated up-front, and will never change through a [`Self::put`] operation.
+    ///
+    /// # Panics
+    ///
+    /// - If the capacity is 0.
+    /// - If the capacity exceeds `isize::MAX` bytes.
+    pub fn new(cap: usize) -> Self {
+        assert_ne!(cap, 0, "The capacity of a cache cannot be 0");
+        Self {
+            // Note: since the element of the vector contains an AtomicU64, it is definitely not zero-sized so cap will never be usize::MAX.
+            data: Vec::with_capacity(cap),
+            generation: AtomicU64::new(0),
+            cap,
+        }
+    }
+
+    /// The capacity of this LRU cache, that is the maximum number of elements it can hold before evicting elements from the cache.
+    ///
+    /// The cache will contain at most this number of elements at any given time.
+    pub fn capacity(&self) -> usize {
+        self.cap
+    }
+
+    fn next_generation(&self) -> u64 {
+        // Acquire so this "happens-before" any potential store to a data cell (with Release ordering)
+        let generation = self.generation.fetch_add(1, Ordering::Acquire);
+        generation + 1
+    }
+
+    fn next_generation_mut(&mut self) -> u64 {
+        let generation = self.generation.get_mut();
+        *generation += 1;
+        *generation
+    }
+
+    /// Add a value in the cache, evicting an older value if necessary.
+    ///
+    /// If a value was evicted from the cache, it is returned.
+    ///
+    /// # Complexity
+    ///
+    /// - If the cache is full, then linear in the capacity.
+    /// - Otherwise constant.
+    pub fn put(&mut self, value: T) -> Option<T> {
+        // no need for a memory fence: we assume that whichever mechanism provides us synchronization
+        // (very probably, a RwLock) takes care of fencing for us.
+
+        let next_generation = self.next_generation_mut();
+        let evicted = if self.is_full() { self.pop() } else { None };
+        self.data.push((AtomicU64::new(next_generation), value));
+        evicted
+    }
+
+    /// Evict the oldest value from the cache.
+    ///
+    /// If the cache is empty, `None` will be returned.
+    ///
+    /// # Complexity
+    ///
+    /// - Linear in the capacity of the cache.
+    pub fn pop(&mut self) -> Option<T> {
+        // Iterator::min_by_key provides shared references to its elements,
+        // but we need (and can afford!) an exclusive one, so let's make an explicit loop
+        let mut min_generation_index = None;
+        for (index, (generation, _)) in self.data.iter_mut().enumerate() {
+            let generation = *generation.get_mut();
+            if let Some((_, min_generation)) = min_generation_index {
+                if min_generation > generation {
+                    min_generation_index = Some((index, generation));
+                }
+            } else {
+                min_generation_index = Some((index, generation))
+            }
+        }
+        min_generation_index.map(|(min_index, _)| self.data.swap_remove(min_index).1)
+    }
+
+    /// The current number of elements in the cache.
+    ///
+    /// This value is guaranteed to be less than or equal to [`Self::capacity`].
+    pub fn len(&self) -> usize {
+        self.data.len()
+    }
+
+    /// Returns `true` if putting any additional element in the cache would cause the eviction of an element.
+    pub fn is_full(&self) -> bool {
+        self.len() == self.capacity()
+    }
+}
+
+pub struct LruMap<K, V>(Lru<(K, V)>);
+
+impl<K, V> LruMap<K, V>
+where
+    K: Eq,
+{
+    /// Creates a new LRU cache map with the specified capacity.
+    ///
+    /// The capacity is allocated up-front, and will never change through a [`Self::insert`] operation.
+    ///
+    /// # Panics
+    ///
+    /// - If the capacity is 0.
+    /// - If the capacity exceeds `isize::MAX` bytes.
+    pub fn new(cap: usize) -> Self {
+        Self(Lru::new(cap))
+    }
+
+    /// Gets a value in the cache map by its key.
+    ///
+    /// If no value matches, `None` will be returned.
+    ///
+    /// # Complexity
+    ///
+    /// - Linear in the capacity of the cache.
+    pub fn get(&self, key: &K) -> Option<&V> {
+        for (generation, (candidate, value)) in self.0.data.iter() {
+            if key == candidate {
+                generation.store(self.0.next_generation(), Ordering::Release);
+                return Some(value);
+            }
+        }
+        None
+    }
+
+    /// Gets a value in the cache map by its key.
+    ///
+    /// If no value matches, `None` will be returned.
+    ///
+    /// # Complexity
+    ///
+    /// - Linear in the capacity of the cache.
+    pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {
+        let next_generation = self.0.next_generation_mut();
+        for (generation, (candidate, value)) in self.0.data.iter_mut() {
+            if key == candidate {
+                *generation.get_mut() = next_generation;
+                return Some(value);
+            }
+        }
+        None
+    }
+
+    /// Inserts a value in the cache map by its key, replacing any existing value and returning any evicted value.
+    ///
+    /// # Complexity
+    ///
+    /// - Linear in the capacity of the cache.
+    pub fn insert(&mut self, key: K, mut value: V) -> InsertionOutcome<K, V> {
+        match self.get_mut(&key) {
+            Some(old_value) => {
+                std::mem::swap(old_value, &mut value);
+                InsertionOutcome::Replaced(value)
+            }
+            None => match self.0.put((key, value)) {
+                Some((key, value)) => InsertionOutcome::Evicted(key, value),
+                None => InsertionOutcome::InsertedNew,
+            },
+        }
+    }
+
+    /// Removes an element from the cache map by its key, returning its value.
+    ///
+    /// Returns `None` if there was no element with this key in the cache.
+    ///
+    /// # Complexity
+    ///
+    /// - Linear in the capacity of the cache.
+    pub fn remove(&mut self, key: &K) -> Option<V> {
+        for (index, (_, (candidate, _))) in self.0.data.iter_mut().enumerate() {
+            if key == candidate {
+                return Some(self.0.data.swap_remove(index).1 .1);
+            }
+        }
+        None
+    }
+}
+
+/// The result of an insertion in a LRU map.
+pub enum InsertionOutcome<K, V> {
+    /// The key was not in the cache, the key-value pair has been inserted.
+    InsertedNew,
+    /// The key was not in the cache and an old key-value pair was evicted from the cache to make room for its insertions.
+    Evicted(K, V),
+    /// The key was already in the cache map, its value has been updated.
+    Replaced(V),
+}

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap

@@ -1,42 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { received_document_ids: 2, deleted_documents: Some(2) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-"documentDeletion" [1,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-["doggos"]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/documents.snap

@@ -1,9 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-[
-  {
-    "id": 3,
-    "doggo": "bork"
-  }
-]

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap

@@ -1,37 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,]
-----------------------------------------------------------------------
-### Index Mapper:
-[]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap

@@ -1,40 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [0,]
-"documentDeletion" [1,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-[]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_failing_the_deletion.snap

@@ -1,43 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [1,]
-failed [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-[]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/after_last_successful_addition.snap

@@ -1,45 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: failed, error: ResponseError { code: 200, message: "Index `doggos` not found.", error_code: "index_not_found", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_not_found" }, details: { received_document_ids: 2, deleted_documents: Some(0) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [1,]
-failed [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-["doggos"]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/documents.snap

@@ -1,17 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-[
-  {
-    "id": 1,
-    "doggo": "jean bob"
-  },
-  {
-    "id": 2,
-    "catto": "jorts"
-  },
-  {
-    "id": 3,
-    "doggo": "bork"
-  }
-]

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_first_task.snap

@@ -1,36 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,]
-----------------------------------------------------------------------
-### Index Mapper:
-[]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------
-

index-scheduler/src/snapshots/lib.rs/document_deletion_and_document_addition/registered_the_second_task.snap

@@ -1,40 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { received_document_ids: 2, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1", "2"] }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-[]
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------
-

index-scheduler/src/utils.rs

@@ -439,29 +439,20 @@ impl IndexScheduler {
                         provided_ids: received_document_ids,
                         deleted_documents,
                     } => {
-                        assert_eq!(kind.as_kind(), Kind::DocumentDeletion);
-                        let (index_uid, documents_ids) =
-                            if let KindWithContent::DocumentDeletion {
-                                ref index_uid,
-                                ref documents_ids,
-                            } = kind
-                            {
-                                (index_uid, documents_ids)
-                            } else {
-                                unreachable!()
-                            };
-                        assert_eq!(&task_index_uid.unwrap(), index_uid);
+                        if let Some(deleted_documents) = deleted_documents {
+                            assert_eq!(status, Status::Succeeded);
+                            assert!(deleted_documents <= received_document_ids as u64);
+                            assert_eq!(kind.as_kind(), Kind::DocumentDeletion);
 
-                        match status {
-                            Status::Enqueued | Status::Processing => (),
-                            Status::Succeeded => {
-                                assert!(deleted_documents.unwrap() <= received_document_ids as u64);
-                                assert!(documents_ids.len() == received_document_ids);
-                            }
-                            Status::Failed | Status::Canceled => {
-                                assert!(deleted_documents == Some(0));
-                                assert!(documents_ids.len() == received_document_ids);
+                            match &kind {
+                                KindWithContent::DocumentDeletion { index_uid, documents_ids } => {
+                                    assert_eq!(&task_index_uid.unwrap(), index_uid);
+                                    assert!(documents_ids.len() >= received_document_ids);
+                                }
+                                _ => panic!(),
                             }
+                        } else {
+                            assert_ne!(status, Status::Succeeded);
                         }
                     }
                     Details::ClearAll { deleted_documents } => {
@@ -538,3 +529,37 @@ impl IndexScheduler {
         }
     }
 }
+
+pub fn dichotomic_search(start_point: usize, mut is_good: impl FnMut(usize) -> bool) -> usize {
+    let mut biggest_good = None;
+    let mut smallest_bad = None;
+    let mut current = start_point;
+    loop {
+        let is_good = is_good(current);
+
+        (biggest_good, smallest_bad, current) = match (biggest_good, smallest_bad, is_good) {
+            (None, None, false) => (None, Some(current), current / 2),
+            (None, None, true) => (Some(current), None, current * 2),
+            (None, Some(smallest_bad), true) => {
+                (Some(current), Some(smallest_bad), (current + smallest_bad) / 2)
+            }
+            (None, Some(_), false) => (None, Some(current), current / 2),
+            (Some(_), None, true) => (Some(current), None, current * 2),
+            (Some(biggest_good), None, false) => {
+                (Some(biggest_good), Some(current), (biggest_good + current) / 2)
+            }
+            (Some(_), Some(smallest_bad), true) => {
+                (Some(current), Some(smallest_bad), (smallest_bad + current) / 2)
+            }
+            (Some(biggest_good), Some(_), false) => {
+                (Some(biggest_good), Some(current), (biggest_good + current) / 2)
+            }
+        };
+        if current == 0 {
+            return current;
+        }
+        if smallest_bad.is_some() && biggest_good.is_some() && biggest_good >= Some(current) {
+            return current;
+        }
+    }
+}

json-depth-checker/Cargo.toml

@@ -1,16 +1,10 @@
 [package]
 name = "json-depth-checker"
+version = "1.0.0"
+edition = "2021"
 description = "A library that indicates if a JSON must be flattened"
 publish = false
 
-version.workspace = true
-authors.workspace = true
-# description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
 [dependencies]
 serde_json = "1.0"
 

meili-snap/Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "meili-snap"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.0"
+edition = "2021"
 
 [dependencies]
 insta = { version = "^1.19.1", features = ["json", "redactions"] }

meilisearch-auth/Cargo.toml

@@ -1,14 +1,7 @@
 [package]
 name = "meilisearch-auth"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.0"
+edition = "2021"
 
 [dependencies]
 base64 = "0.13.1"

meilisearch-types/Cargo.toml

@@ -1,21 +1,15 @@
 [package]
 name = "meilisearch-types"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.0"
+authors = ["marin <postma.marin@protonmail.com>"]
+edition = "2021"
 
 [dependencies]
 actix-web = { version = "4.2.1", default-features = false }
 anyhow = "1.0.65"
 convert_case = "0.6.0"
 csv = "1.1.6"
-deserr = "0.5.0"
+deserr = "0.4.1"
 either = { version = "1.6.1", features = ["serde"] }
 enum-iterator = "1.1.3"
 file-store = { path = "../file-store" }

meilisearch-types/src/document_formats.rs

@@ -19,7 +19,7 @@ type Result<T> = std::result::Result<T, DocumentFormatError>;
 pub enum PayloadType {
     Ndjson,
     Json,
-    Csv { delimiter: u8 },
+    Csv,
 }
 
 impl fmt::Display for PayloadType {
@@ -27,7 +27,7 @@ impl fmt::Display for PayloadType {
         match self {
             PayloadType::Ndjson => f.write_str("ndjson"),
             PayloadType::Json => f.write_str("json"),
-            PayloadType::Csv { .. } => f.write_str("csv"),
+            PayloadType::Csv => f.write_str("csv"),
         }
     }
 }
@@ -105,11 +105,11 @@ impl ErrorCode for DocumentFormatError {
 }
 
 /// Reads CSV from input and write an obkv batch to writer.
-pub fn read_csv(file: &File, writer: impl Write + Seek, delimiter: u8) -> Result<u64> {
+pub fn read_csv(file: &File, writer: impl Write + Seek) -> Result<u64> {
     let mut builder = DocumentsBatchBuilder::new(writer);
     let mmap = unsafe { MmapOptions::new().map(file)? };
-    let csv = csv::ReaderBuilder::new().delimiter(delimiter).from_reader(mmap.as_ref());
-    builder.append_csv(csv).map_err(|e| (PayloadType::Csv { delimiter }, e))?;
+    let csv = csv::Reader::from_reader(mmap.as_ref());
+    builder.append_csv(csv).map_err(|e| (PayloadType::Csv, e))?;
 
     let count = builder.documents_count();
     let _ = builder.into_inner().map_err(DocumentFormatError::Io)?;

meilisearch-types/src/error.rs

@@ -220,7 +220,6 @@ InvalidDocumentOffset                 , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexCsvDelimiter              , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;

meilisearch/Cargo.toml

@@ -1,16 +1,10 @@
 [package]
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+description = "Meilisearch HTTP server"
+edition = "2021"
+license = "MIT"
 name = "meilisearch"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-default-run = "meilisearch"
+version = "1.0.0"
 
 [dependencies]
 actix-cors = "0.6.3"
@@ -25,7 +19,7 @@ byte-unit = { version = "4.0.14", default-features = false, features = ["std", "
 bytes = "1.2.1"
 clap = { version = "4.0.9", features = ["derive", "env"] }
 crossbeam-channel = "0.5.6"
-deserr = "0.5.0"
+deserr = "0.4.1"
 dump = { path = "../dump" }
 either = "1.8.0"
 env_logger = "0.9.1"
@@ -96,7 +90,7 @@ yaup = "0.2.1"
 
 [build-dependencies]
 anyhow = { version = "1.0.65", optional = true }
-cargo_toml = { version = "0.14.0", optional = true }
+cargo_toml = { version = "0.13.0", optional = true }
 hex = { version = "0.4.3", optional = true }
 reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
 sha-1 = { version = "0.10.0", optional = true }
@@ -116,5 +110,5 @@ japanese = ["meilisearch-types/japanese"]
 thai = ["meilisearch-types/thai"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.6/build.zip"
-sha1 = "dce0aba16bceab5549edf9f01de89858800f7422"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.5/build.zip"
+sha1 = "6fe959b78511b32e9ff857fd9fd31740633b9fce"

meilisearch/src/analytics/segment_analytics.rs

@@ -284,7 +284,8 @@ impl From<Opt> for Infos {
             ScheduleSnapshot::Enabled(interval) => Some(interval),
         };
 
-        let IndexerOpts { max_indexing_memory, max_indexing_threads } = indexer_options;
+        let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } =
+            indexer_options;
 
         // We're going to override every sensible information.
         // We consider information sensible if it contains a path, an address, or a key.

meilisearch/src/error.rs

@@ -11,8 +11,6 @@ pub enum MeilisearchHttpError {
     #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}",
             .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", "))]
     MissingContentType(Vec<String>),
-    #[error("The Content-Type `{0}` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.")]
-    CsvDelimiterWithWrongContentType(String),
     #[error(
         "The Content-Type `{0}` is invalid. Accepted values for the Content-Type header are: {}",
         .1.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
@@ -54,7 +52,6 @@ impl ErrorCode for MeilisearchHttpError {
     fn error_code(&self) -> Code {
         match self {
             MeilisearchHttpError::MissingContentType(_) => Code::MissingContentType,
-            MeilisearchHttpError::CsvDelimiterWithWrongContentType(_) => Code::InvalidContentType,
             MeilisearchHttpError::MissingPayload(_) => Code::MissingPayload,
             MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,
             MeilisearchHttpError::DocumentNotFound(_) => Code::DocumentNotFound,

meilisearch/src/lib.rs

@@ -45,6 +45,33 @@ use option::ScheduleSnapshot;
 
 use crate::error::MeilisearchHttpError;
 
+/// Default number of simultaneously opened indexes,
+/// lower for Windows that dedicates a smaller virtual address space to processes.
+///
+/// The value was chosen this way:
+///
+/// - Windows provides a small virtual address space of about 10TiB to processes.
+/// - The chosen value allows for indexes to reach a safe size of 1TiB.
+/// - This can accomodate an unlimited number of indexes as long as they stay below 1TiB size.
+#[cfg(windows)]
+const DEFAULT_INDEX_COUNT: usize = 10;
+/// Default number of simultaneously opened indexes.
+///
+/// The higher, the better for avoiding reopening indexes.
+///
+/// The value was chosen this way:
+///
+/// - Opening an index consumes a file descriptor.
+/// - The default on many unices is about 256 file descriptors for a process.
+/// - 100 is a little bit less than half this value.
+///
+/// In the future, this value could be computed from the dynamic number of allowed file descriptors for the current process.
+///
+/// On Unices, this value is largely irrelevant to virtual address space, because due to index resizing the indexes should take virtual memory in the same ballpark
+/// as their disk size and it is unlikely for a user to have a sum of index weighing 128TB on a single Meilisearch node.
+#[cfg(not(windows))]
+const DEFAULT_INDEX_COUNT: usize = 100;
+
 /// Check if a db is empty. It does not provide any information on the
 /// validity of the data in it.
 /// We consider a database as non empty when it's a non empty directory.
@@ -205,9 +232,11 @@ fn open_or_create_database_unchecked(
             snapshots_path: opt.snapshot_dir.clone(),
             dumps_path: opt.dump_dir.clone(),
             task_db_size: opt.max_task_db_size.get_bytes() as usize,
-            index_size: opt.max_index_size.get_bytes() as usize,
+            index_base_map_size: opt.max_index_size.get_bytes() as usize,
             indexer_config: (&opt.indexer_options).try_into()?,
             autobatching_enabled: true,
+            index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
+            index_count: DEFAULT_INDEX_COUNT,
         })?)
     };
 

meilisearch/src/option.rs

@@ -65,11 +65,10 @@ const MEILI_MAX_INDEXING_THREADS: &str = "MEILI_MAX_INDEXING_THREADS";
 const DEFAULT_LOG_EVERY_N: usize = 100_000;
 
 // Each environment (index and task-db) is taking space in the virtual address space.
-//
-// The size of the virtual address space is limited by the OS. About 100TB for Linux and about 10TB for Windows.
-// This means that the number of indexes is limited to about 200 for Linux and about 20 for Windows.
-pub const INDEX_SIZE: u64 = 536_870_912_000; // 500 GiB
-pub const TASK_DB_SIZE: u64 = 10_737_418_240; // 10 GiB
+// When creating a new environment, it starts its life with 10GiB of virtual address space.
+// It is then later resized if needs be.
+pub const INDEX_SIZE: u64 = 2 * 1024 * 1024 * 1024 * 1024; // 2 TiB
+pub const TASK_DB_SIZE: u64 = 10 * 1024 * 1024 * 1024; // 10 GiB
 
 #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
 #[serde(rename_all = "UPPERCASE")]
@@ -494,12 +493,21 @@ pub struct IndexerOpts {
     #[clap(long, env = MEILI_MAX_INDEXING_THREADS, default_value_t)]
     #[serde(default)]
     pub max_indexing_threads: MaxThreads,
+
+    /// Whether or not we want to determine the budget of virtual memory address space we have available dynamically
+    /// (the default), or statically.
+    ///
+    /// Determining the budget of virtual memory address space dynamically takes some time on some systems (such as macOS)
+    /// and may make tests non-deterministic, so we want to skip it in tests.
+    #[clap(skip)]
+    #[serde(skip)]
+    pub skip_index_budget: bool,
 }
 
 impl IndexerOpts {
     /// Exports the values to their corresponding env vars if they are not set.
     pub fn export_to_env(self) {
-        let IndexerOpts { max_indexing_memory, max_indexing_threads } = self;
+        let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } = self;
         if let Some(max_indexing_memory) = max_indexing_memory.0 {
             export_to_env_if_not_present(
                 MEILI_MAX_INDEXING_MEMORY,
@@ -527,6 +535,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
             max_memory: other.max_indexing_memory.map(|b| b.get_bytes() as usize),
             thread_pool: Some(thread_pool),
             max_positions_per_attributes: None,
+            skip_index_budget: other.skip_index_budget,
             ..Default::default()
         })
     }

meilisearch/src/routes/indexes/documents.rs

@@ -10,10 +10,10 @@ use futures::StreamExt;
 use index_scheduler::IndexScheduler;
 use log::debug;
 use meilisearch_types::deserr::query_params::Param;
-use meilisearch_types::deserr::DeserrQueryParamError;
+use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::document_formats::{read_csv, read_json, read_ndjson, PayloadType};
 use meilisearch_types::error::deserr_codes::*;
-use meilisearch_types::error::{Code, ResponseError};
+use meilisearch_types::error::ResponseError;
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
@@ -67,7 +67,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
             .route(web::get().to(SeqHandler(get_all_documents)))
-            .route(web::post().to(SeqHandler(replace_documents)))
+            .route(web::post().to(SeqHandler(add_documents)))
             .route(web::put().to(SeqHandler(update_documents)))
             .route(web::delete().to(SeqHandler(clear_all_documents))),
     )
@@ -156,31 +156,16 @@ pub async fn get_all_documents(
 }
 
 #[derive(Deserialize, Debug, Deserr)]
-#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct UpdateDocumentsQuery {
-    #[deserr(default, error = DeserrQueryParamError<InvalidIndexPrimaryKey>)]
+    #[deserr(default, error = DeserrJsonError<InvalidIndexPrimaryKey>)]
     pub primary_key: Option<String>,
-    #[deserr(default, try_from(char) = from_char_csv_delimiter -> DeserrQueryParamError<InvalidIndexCsvDelimiter>, error = DeserrQueryParamError<InvalidIndexCsvDelimiter>)]
-    pub csv_delimiter: Option<u8>,
 }
 
-fn from_char_csv_delimiter(
-    c: char,
-) -> Result<Option<u8>, DeserrQueryParamError<InvalidIndexCsvDelimiter>> {
-    if c.is_ascii() {
-        Ok(Some(c as u8))
-    } else {
-        Err(DeserrQueryParamError::new(
-            format!("csv delimiter must be an ascii character. Found: `{}`", c),
-            Code::InvalidIndexCsvDelimiter,
-        ))
-    }
-}
-
-pub async fn replace_documents(
+pub async fn add_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
-    params: AwebQueryParameter<UpdateDocumentsQuery, DeserrQueryParamError>,
+    params: AwebQueryParameter<UpdateDocumentsQuery, DeserrJsonError>,
     body: Payload,
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
@@ -198,7 +183,6 @@ pub async fn replace_documents(
         index_scheduler,
         index_uid,
         params.primary_key,
-        params.csv_delimiter,
         body,
         IndexDocumentsMethod::ReplaceDocuments,
         allow_index_creation,
@@ -211,7 +195,7 @@ pub async fn replace_documents(
 pub async fn update_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
-    params: AwebQueryParameter<UpdateDocumentsQuery, DeserrQueryParamError>,
+    params: AwebQueryParameter<UpdateDocumentsQuery, DeserrJsonError>,
     body: Payload,
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
@@ -219,7 +203,6 @@ pub async fn update_documents(
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     debug!("called with params: {:?}", params);
-    let params = params.into_inner();
 
     analytics.update_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
@@ -228,8 +211,7 @@ pub async fn update_documents(
         extract_mime_type(&req)?,
         index_scheduler,
         index_uid,
-        params.primary_key,
-        params.csv_delimiter,
+        params.into_inner().primary_key,
         body,
         IndexDocumentsMethod::UpdateDocuments,
         allow_index_creation,
@@ -239,43 +221,26 @@ pub async fn update_documents(
     Ok(HttpResponse::Accepted().json(task))
 }
 
-#[allow(clippy::too_many_arguments)]
 async fn document_addition(
     mime_type: Option<Mime>,
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, Data<IndexScheduler>>,
     index_uid: IndexUid,
     primary_key: Option<String>,
-    csv_delimiter: Option<u8>,
     mut body: Payload,
     method: IndexDocumentsMethod,
     allow_index_creation: bool,
 ) -> Result<SummarizedTaskView, MeilisearchHttpError> {
-    let format = match (
-        mime_type.as_ref().map(|m| (m.type_().as_str(), m.subtype().as_str())),
-        csv_delimiter,
-    ) {
-        (Some(("application", "json")), None) => PayloadType::Json,
-        (Some(("application", "x-ndjson")), None) => PayloadType::Ndjson,
-        (Some(("text", "csv")), None) => PayloadType::Csv { delimiter: b',' },
-        (Some(("text", "csv")), Some(delimiter)) => PayloadType::Csv { delimiter },
-
-        (Some(("application", "json")), Some(_)) => {
-            return Err(MeilisearchHttpError::CsvDelimiterWithWrongContentType(String::from(
-                "application/json",
-            )))
-        }
-        (Some(("application", "x-ndjson")), Some(_)) => {
-            return Err(MeilisearchHttpError::CsvDelimiterWithWrongContentType(String::from(
-                "application/x-ndjson",
-            )))
-        }
-        (Some((type_, subtype)), _) => {
+    let format = match mime_type.as_ref().map(|m| (m.type_().as_str(), m.subtype().as_str())) {
+        Some(("application", "json")) => PayloadType::Json,
+        Some(("application", "x-ndjson")) => PayloadType::Ndjson,
+        Some(("text", "csv")) => PayloadType::Csv,
+        Some((type_, subtype)) => {
             return Err(MeilisearchHttpError::InvalidContentType(
                 format!("{}/{}", type_, subtype),
                 ACCEPTED_CONTENT_TYPE.clone(),
             ))
         }
-        (None, _) => {
+        None => {
             return Err(MeilisearchHttpError::MissingContentType(ACCEPTED_CONTENT_TYPE.clone()))
         }
     };
@@ -320,9 +285,7 @@ async fn document_addition(
     let documents_count = tokio::task::spawn_blocking(move || {
         let documents_count = match format {
             PayloadType::Json => read_json(&read_file, update_file.as_file_mut())?,
-            PayloadType::Csv { delimiter } => {
-                read_csv(&read_file, update_file.as_file_mut(), delimiter)?
-            }
+            PayloadType::Csv => read_csv(&read_file, update_file.as_file_mut())?,
             PayloadType::Ndjson => read_ndjson(&read_file, update_file.as_file_mut())?,
         };
         // we NEED to persist the file here because we moved the `udpate_file` in another task.

meilisearch/tests/common/index.rs

@@ -30,7 +30,7 @@ impl Index<'_> {
             .post_str(
                 url,
                 include_str!("../assets/test_set.json"),
-                vec![("content-type", "application/json")],
+                ("content-type", "application/json"),
             )
             .await;
         assert_eq!(code, 202);
@@ -46,7 +46,7 @@ impl Index<'_> {
             .post_str(
                 url,
                 include_str!("../assets/test_set.ndjson"),
-                vec![("content-type", "application/x-ndjson")],
+                ("content-type", "application/x-ndjson"),
             )
             .await;
         assert_eq!(code, 202);
@@ -96,21 +96,6 @@ impl Index<'_> {
         self.service.post_encoded(url, documents, self.encoder).await
     }
 
-    pub async fn raw_add_documents(
-        &self,
-        payload: &str,
-        content_type: Option<&str>,
-        query_parameter: &str,
-    ) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), query_parameter);
-
-        if let Some(content_type) = content_type {
-            self.service.post_str(url, payload, vec![("Content-Type", content_type)]).await
-        } else {
-            self.service.post_str(url, payload, Vec::new()).await
-        }
-    }
-
     pub async fn update_documents(
         &self,
         documents: Value,
@@ -125,21 +110,6 @@ impl Index<'_> {
         self.service.put_encoded(url, documents, self.encoder).await
     }
 
-    pub async fn raw_update_documents(
-        &self,
-        payload: &str,
-        content_type: Option<&str>,
-        query_parameter: &str,
-    ) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), query_parameter);
-
-        if let Some(content_type) = content_type {
-            self.service.put_str(url, payload, vec![("Content-Type", content_type)]).await
-        } else {
-            self.service.put_str(url, payload, Vec::new()).await
-        }
-    }
-
     pub async fn wait_task(&self, update_id: u64) -> Value {
         // try several times to get status, or panic to not wait forever
         let url = format!("/tasks/{}", update_id);

meilisearch/tests/common/server.rs

@@ -201,6 +201,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         indexer_options: IndexerOpts {
             // memory has to be unlimited because several meilisearch are running in test context.
             max_indexing_memory: MaxMemory::unlimited(),
+            skip_index_budget: true,
             ..Parser::parse_from(None as Option<&str>)
         },
         #[cfg(feature = "metrics")]

meilisearch/tests/common/service.rs

@@ -34,18 +34,17 @@ impl Service {
         self.request(req).await
     }
 
-    /// Send a test post request from a text body.
+    /// Send a test post request from a text body, with a `content-type:application/json` header.
     pub async fn post_str(
         &self,
         url: impl AsRef<str>,
         body: impl AsRef<str>,
-        headers: Vec<(&str, &str)>,
+        header: (&str, &str),
     ) -> (Value, StatusCode) {
-        let mut req =
-            test::TestRequest::post().uri(url.as_ref()).set_payload(body.as_ref().to_string());
-        for header in headers {
-            req = req.insert_header(header);
-        }
+        let req = test::TestRequest::post()
+            .uri(url.as_ref())
+            .set_payload(body.as_ref().to_string())
+            .insert_header(header);
         self.request(req).await
     }
 
@@ -58,21 +57,6 @@ impl Service {
         self.put_encoded(url, body, Encoder::Plain).await
     }
 
-    /// Send a test put request from a text body.
-    pub async fn put_str(
-        &self,
-        url: impl AsRef<str>,
-        body: impl AsRef<str>,
-        headers: Vec<(&str, &str)>,
-    ) -> (Value, StatusCode) {
-        let mut req =
-            test::TestRequest::put().uri(url.as_ref()).set_payload(body.as_ref().to_string());
-        for header in headers {
-            req = req.insert_header(header);
-        }
-        self.request(req).await
-    }
-
     pub async fn put_encoded(
         &self,
         url: impl AsRef<str>,

meilisearch/tests/documents/add_documents.rs

@@ -216,133 +216,6 @@ async fn add_single_document_with_every_encoding() {
     }
 }
 
-#[actix_rt::test]
-async fn add_csv_document() {
-    let server = Server::new().await;
-    let index = server.index("pets");
-
-    let document = "#id,name,race
-0,jean,bernese mountain
-1,jorts,orange cat";
-
-    let (response, code) = index.raw_update_documents(document, Some("text/csv"), "").await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 0,
-      "indexUid": "pets",
-      "status": "enqueued",
-      "type": "documentAdditionOrUpdate",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let response = index.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
-    {
-      "uid": 0,
-      "indexUid": "pets",
-      "status": "succeeded",
-      "type": "documentAdditionOrUpdate",
-      "canceledBy": null,
-      "details": {
-        "receivedDocuments": 2,
-        "indexedDocuments": 2
-      },
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-
-    let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
-    snapshot!(code, @"200 OK");
-    snapshot!(json_string!(documents), @r###"
-    {
-      "results": [
-        {
-          "#id": "0",
-          "name": "jean",
-          "race": "bernese mountain"
-        },
-        {
-          "#id": "1",
-          "name": "jorts",
-          "race": "orange cat"
-        }
-      ],
-      "offset": 0,
-      "limit": 20,
-      "total": 2
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn add_csv_document_with_custom_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("pets");
-
-    let document = "#id|name|race
-0|jean|bernese mountain
-1|jorts|orange cat";
-
-    let (response, code) =
-        index.raw_update_documents(document, Some("text/csv"), "?csvDelimiter=|").await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 0,
-      "indexUid": "pets",
-      "status": "enqueued",
-      "type": "documentAdditionOrUpdate",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let response = index.wait_task(response["taskUid"].as_u64().unwrap()).await;
-    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
-    {
-      "uid": 0,
-      "indexUid": "pets",
-      "status": "succeeded",
-      "type": "documentAdditionOrUpdate",
-      "canceledBy": null,
-      "details": {
-        "receivedDocuments": 2,
-        "indexedDocuments": 2
-      },
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-
-    let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
-    snapshot!(code, @"200 OK");
-    snapshot!(json_string!(documents), @r###"
-    {
-      "results": [
-        {
-          "#id": "0",
-          "name": "jean",
-          "race": "bernese mountain"
-        },
-        {
-          "#id": "1",
-          "name": "jorts",
-          "race": "orange cat"
-        }
-      ],
-      "offset": 0,
-      "limit": 20,
-      "total": 2
-    }
-    "###);
-}
-
 /// any other content-type is must be refused
 #[actix_rt::test]
 async fn error_add_documents_test_bad_content_types() {
@@ -1154,53 +1027,6 @@ async fn error_document_field_limit_reached() {
         @"");
 }
 
-#[actix_rt::test]
-async fn add_documents_with_geo_field() {
-    let server = Server::new().await;
-    let index = server.index("doggo");
-    index.update_settings(json!({"sortableAttributes": ["_geo"]})).await;
-
-    let documents = json!([
-        {
-            "id": "1",
-        },
-        {
-            "id": "2",
-            "_geo": null,
-        },
-        {
-            "id": "3",
-            "_geo": { "lat": 1, "lng": 1 },
-        },
-        {
-            "id": "4",
-            "_geo": { "lat": "1", "lng": "1" },
-        },
-    ]);
-
-    index.add_documents(documents, None).await;
-    let response = index.wait_task(1).await;
-    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }),
-        @r###"
-    {
-      "uid": 1,
-      "indexUid": "doggo",
-      "status": "succeeded",
-      "type": "documentAdditionOrUpdate",
-      "canceledBy": null,
-      "details": {
-        "receivedDocuments": 4,
-        "indexedDocuments": 4
-      },
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-}
-
 #[actix_rt::test]
 async fn add_documents_invalid_geo_field() {
     let server = Server::new().await;

meilisearch/tests/documents/errors.rs

@@ -1,6 +1,5 @@
 use meili_snap::*;
 use serde_json::json;
-use urlencoding::encode;
 
 use crate::common::Server;
 
@@ -98,323 +97,3 @@ async fn delete_documents_batch() {
     }
     "###);
 }
-
-#[actix_rt::test]
-async fn replace_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_add_documents("", Some("application/json"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A json payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-
-    let (response, code) = index.raw_add_documents("", Some("application/x-ndjson"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A ndjson payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-
-    let (response, code) = index.raw_add_documents("", Some("text/csv"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A csv payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn update_documents_missing_payload() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_update_documents("", Some("application/json"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A json payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-
-    let (response, code) = index.raw_update_documents("", Some("application/x-ndjson"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A ndjson payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-
-    let (response, code) = index.raw_update_documents("", Some("text/csv"), "").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A csv payload is missing.",
-      "code": "missing_payload",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_payload"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn replace_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_add_documents("", None, "").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "missing_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_content_type"
-    }
-    "###);
-
-    // even with a csv delimiter specified this error is triggered first
-    let (response, code) = index.raw_add_documents("", None, "?csvDelimiter=;").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "missing_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_content_type"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn update_documents_missing_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_update_documents("", None, "").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "missing_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_content_type"
-    }
-    "###);
-
-    // even with a csv delimiter specified this error is triggered first
-    let (response, code) = index.raw_update_documents("", None, "?csvDelimiter=;").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "missing_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#missing_content_type"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn replace_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_add_documents("", Some("doggo"), "").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `doggo` is invalid. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn update_documents_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) = index.raw_update_documents("", Some("doggo"), "").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `doggo` is invalid. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn replace_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "Invalid value in parameter `csvDelimiter`: expected a string of one character, but found an empty string",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter=doggo").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "Invalid value in parameter `csvDelimiter`: expected a string of one character, but found the following string of 5 characters: `doggo`",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-
-    let (response, code) = index
-        .raw_add_documents("", Some("application/json"), &format!("?csvDelimiter={}", encode("🍰")))
-        .await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "csv delimiter must be an ascii character. Found: `🍰`",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn update_documents_bad_csv_delimiter() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) =
-        index.raw_update_documents("", Some("application/json"), "?csvDelimiter").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "Invalid value in parameter `csvDelimiter`: expected a string of one character, but found an empty string",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-
-    let (response, code) =
-        index.raw_update_documents("", Some("application/json"), "?csvDelimiter=doggo").await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "Invalid value in parameter `csvDelimiter`: expected a string of one character, but found the following string of 5 characters: `doggo`",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-
-    let (response, code) = index
-        .raw_update_documents(
-            "",
-            Some("application/json"),
-            &format!("?csvDelimiter={}", encode("🍰")),
-        )
-        .await;
-    snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "csv delimiter must be an ascii character. Found: `🍰`",
-      "code": "invalid_index_csv_delimiter",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_index_csv_delimiter"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn replace_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter=a").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `application/json` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-
-    let (response, code) =
-        index.raw_add_documents("", Some("application/x-ndjson"), "?csvDelimiter=a").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `application/x-ndjson` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-}
-
-#[actix_rt::test]
-async fn update_documents_csv_delimiter_with_bad_content_type() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (response, code) =
-        index.raw_update_documents("", Some("application/json"), "?csvDelimiter=a").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `application/json` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-
-    let (response, code) =
-        index.raw_update_documents("", Some("application/x-ndjson"), "?csvDelimiter=a").await;
-    snapshot!(code, @"415 Unsupported Media Type");
-    snapshot!(json_string!(response), @r###"
-    {
-      "message": "The Content-Type `application/x-ndjson` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.",
-      "code": "invalid_content_type",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#invalid_content_type"
-    }
-    "###);
-}

milli/Cargo.toml

@@ -1,15 +1,8 @@
 [package]
 name = "milli"
+version = "1.0.0"
+authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-# edition.workspace = true
-license.workspace = true
 
 [dependencies]
 bimap = { version = "0.6.2", features = ["serde"] }
@@ -19,7 +12,7 @@ byteorder = "1.4.3"
 charabia = { version = "0.7.0", default-features = false }
 concat-arrays = "0.1.2"
 crossbeam-channel = "0.5.6"
-deserr = "0.5.0"
+deserr = "0.4.1"
 either = "1.8.0"
 flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"

milli/src/snapshot_tests.rs

@@ -6,7 +6,7 @@ use roaring::RoaringBitmap;
 
 use crate::facet::FacetType;
 use crate::heed_codec::facet::{FacetGroupKey, FacetGroupValue};
-use crate::{make_db_snap_from_iter, obkv_to_json, ExternalDocumentsIds, Index};
+use crate::{make_db_snap_from_iter, ExternalDocumentsIds, Index};
 
 #[track_caller]
 pub fn default_db_snapshot_settings_for_test(name: Option<&str>) -> (insta::Settings, String) {
@@ -427,26 +427,8 @@ pub fn snap_settings(index: &Index) -> String {
     snap
 }
 
-pub fn snap_documents(index: &Index) -> String {
-    let mut snap = String::new();
-    let rtxn = index.read_txn().unwrap();
-    let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
-    let display = fields_ids_map.ids().collect::<Vec<_>>();
-
-    for document in index.all_documents(&rtxn).unwrap() {
-        let doc = obkv_to_json(&display, &fields_ids_map, document.unwrap().1).unwrap();
-        snap.push_str(&serde_json::to_string(&doc).unwrap());
-        snap.push('\n');
-    }
-
-    snap
-}
-
 #[macro_export]
 macro_rules! full_snap_of_db {
-    ($index:ident, documents) => {{
-        $crate::snapshot_tests::snap_documents(&$index)
-    }};
     ($index:ident, settings) => {{
         $crate::snapshot_tests::snap_settings(&$index)
     }};

milli/src/update/index_documents/enrich.rs

@@ -395,7 +395,6 @@ pub fn validate_geo_from_json(id: &DocumentId, bytes: &[u8]) -> Result<StdResult
             (Some(_), None) => Ok(Err(MissingLongitude { document_id: debug_id() })),
             (None, None) => Ok(Err(MissingLatitudeAndLongitude { document_id: debug_id() })),
         },
-        Value::Null => Ok(Ok(())),
         value => Ok(Err(NotAnObject { document_id: debug_id(), value })),
     }
 }

milli/src/update/index_documents/extract/extract_geo_points.rs

@@ -59,7 +59,6 @@ pub fn extract_geo_points<R: io::Read + io::Seek>(
         } else if lat.is_some() && lng.is_none() {
             return Err(GeoError::MissingLongitude { document_id: document_id() })?;
         }
-        // else => the _geo object was `null`, there is nothing to do
     }
 
     writer_into_reader(writer)

milli/src/update/index_documents/helpers/merge_functions.rs

@@ -6,7 +6,6 @@ use roaring::RoaringBitmap;
 
 use super::read_u32_ne_bytes;
 use crate::heed_codec::CboRoaringBitmapCodec;
-use crate::update::index_documents::transform::Operation;
 use crate::Result;
 
 pub type MergeFn = for<'a> fn(&[u8], &[Cow<'a, [u8]>]) -> Result<Cow<'a, [u8]>>;
@@ -58,6 +57,21 @@ pub fn keep_latest_obkv<'a>(_key: &[u8], obkvs: &[Cow<'a, [u8]>]) -> Result<Cow<
     Ok(obkvs.last().unwrap().clone())
 }
 
+/// Merge all the obks in the order we see them.
+pub fn merge_obkvs<'a>(_key: &[u8], obkvs: &[Cow<'a, [u8]>]) -> Result<Cow<'a, [u8]>> {
+    Ok(obkvs
+        .iter()
+        .cloned()
+        .reduce(|acc, current| {
+            let first = obkv::KvReader::new(&acc);
+            let second = obkv::KvReader::new(&current);
+            let mut buffer = Vec::new();
+            merge_two_obkvs(first, second, &mut buffer);
+            Cow::from(buffer)
+        })
+        .unwrap())
+}
+
 pub fn merge_two_obkvs(base: obkv::KvReaderU16, update: obkv::KvReaderU16, buffer: &mut Vec<u8>) {
     use itertools::merge_join_by;
     use itertools::EitherOrBoth::{Both, Left, Right};
@@ -74,41 +88,6 @@ pub fn merge_two_obkvs(base: obkv::KvReaderU16, update: obkv::KvReaderU16, buffe
     writer.finish().unwrap();
 }
 
-/// Merge all the obks in the order we see them.
-pub fn merge_obkvs_and_operations<'a>(
-    _key: &[u8],
-    obkvs: &[Cow<'a, [u8]>],
-) -> Result<Cow<'a, [u8]>> {
-    // [add, add, delete, add, add]
-    // we can ignore everything that happened before the last delete.
-    let starting_position =
-        obkvs.iter().rposition(|obkv| obkv[0] == Operation::Deletion as u8).unwrap_or(0);
-
-    // [add, add, delete]
-    // if the last operation was a deletion then we simply return the deletion
-    if starting_position == obkvs.len() - 1 && obkvs.last().unwrap()[0] == Operation::Deletion as u8
-    {
-        return Ok(obkvs[obkvs.len() - 1].clone());
-    }
-    let mut buffer = Vec::new();
-
-    // (add, add, delete) [add, add]
-    // in the other case, no deletion will be encountered during the merge
-    let mut ret =
-        obkvs[starting_position..].iter().cloned().fold(Vec::new(), |mut acc, current| {
-            let first = obkv::KvReader::new(&acc);
-            let second = obkv::KvReader::new(&current[1..]);
-            merge_two_obkvs(first, second, &mut buffer);
-
-            // we want the result of the merge into our accumulator
-            std::mem::swap(&mut acc, &mut buffer);
-            acc
-        });
-
-    ret.insert(0, Operation::Addition as u8);
-    Ok(Cow::from(ret))
-}
-
 pub fn merge_cbo_roaring_bitmaps<'a>(
     _key: &[u8],
     values: &[Cow<'a, [u8]>],

milli/src/update/index_documents/helpers/mod.rs

@@ -13,9 +13,9 @@ pub use grenad_helpers::{
     GrenadParameters, MergeableReader,
 };
 pub use merge_functions::{
-    concat_u32s_array, keep_first, keep_latest_obkv, merge_cbo_roaring_bitmaps,
-    merge_obkvs_and_operations, merge_roaring_bitmaps, merge_two_obkvs,
-    roaring_bitmap_from_u32s_array, serialize_roaring_bitmap, MergeFn,
+    concat_u32s_array, keep_first, keep_latest_obkv, merge_cbo_roaring_bitmaps, merge_obkvs,
+    merge_roaring_bitmaps, merge_two_obkvs, roaring_bitmap_from_u32s_array,
+    serialize_roaring_bitmap, MergeFn,
 };
 
 use crate::MAX_WORD_LENGTH;

milli/src/update/index_documents/mod.rs

@@ -79,7 +79,6 @@ pub struct IndexDocuments<'t, 'u, 'i, 'a, FP, FA> {
     progress: FP,
     should_abort: FA,
     added_documents: u64,
-    deleted_documents: u64,
 }
 
 #[derive(Default, Debug, Clone)]
@@ -123,7 +122,6 @@ where
             wtxn,
             index,
             added_documents: 0,
-            deleted_documents: 0,
         })
     }
 
@@ -168,30 +166,6 @@ where
         Ok((self, Ok(indexed_documents)))
     }
 
-    /// Remove a batch of documents from the current builder.
-    ///
-    /// Returns the number of documents deleted from the builder.
-    pub fn remove_documents(
-        mut self,
-        to_delete: Vec<String>,
-    ) -> Result<(Self, StdResult<u64, UserError>)> {
-        // Early return when there is no document to add
-        if to_delete.is_empty() {
-            return Ok((self, Ok(0)));
-        }
-
-        let deleted_documents = self
-            .transform
-            .as_mut()
-            .expect("Invalid document deletion state")
-            .remove_documents(to_delete, self.wtxn, &self.should_abort)?
-            as u64;
-
-        self.deleted_documents += deleted_documents;
-
-        Ok((self, Ok(deleted_documents)))
-    }
-
     #[logging_timer::time("IndexDocuments::{}")]
     pub fn execute(mut self) -> Result<DocumentAdditionResult> {
         if self.added_documents == 0 {
@@ -1905,328 +1879,4 @@ mod tests {
 
         index.add_documents(doc1).unwrap();
     }
-
-    #[test]
-    fn add_and_delete_documents_in_single_transform() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let documents = documents!([
-            { "id": 1, "doggo": "kevin" },
-            { "id": 2, "doggo": { "name": "bob", "age": 20 } },
-            { "id": 3, "name": "jean", "age": 25 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"3");
-
-        let (builder, removed) = builder.remove_documents(vec![S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"1");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 3,
-            number_of_documents: 2,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":1,"doggo":"kevin"}
-        {"id":3,"name":"jean","age":25}
-        "###);
-    }
-
-    #[test]
-    fn add_update_and_delete_documents_in_single_transform() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let documents = documents!([
-            { "id": 1, "doggo": "kevin" },
-            { "id": 2, "doggo": { "name": "bob", "age": 20 } },
-            { "id": 3, "name": "jean", "age": 25 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"3");
-
-        let documents = documents!([
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "legs": 4 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"2");
-
-        let (builder, removed) = builder.remove_documents(vec![S("1"), S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"2");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 5,
-            number_of_documents: 1,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":3,"name":"jean","age":25,"legs":4}
-        "###);
-    }
-
-    #[test]
-    fn add_document_and_in_another_transform_update_and_delete_documents() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let documents = documents!([
-            { "id": 1, "doggo": "kevin" },
-            { "id": 2, "doggo": { "name": "bob", "age": 20 } },
-            { "id": 3, "name": "jean", "age": 25 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"3");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 3,
-            number_of_documents: 3,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":1,"doggo":"kevin"}
-        {"id":2,"doggo":{"name":"bob","age":20}}
-        {"id":3,"name":"jean","age":25}
-        "###);
-
-        // A first batch of documents has been inserted
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let documents = documents!([
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "legs": 4 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"2");
-
-        let (builder, removed) = builder.remove_documents(vec![S("1"), S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"2");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 2,
-            number_of_documents: 1,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":3,"name":"jean","age":25,"legs":4}
-        "###);
-    }
-
-    #[test]
-    fn delete_document_and_then_add_documents_in_the_same_transform() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let (builder, removed) = builder.remove_documents(vec![S("1"), S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"0");
-
-        let documents = documents!([
-            { "id": 2, "doggo": { "name": "jean", "age": 20 } },
-            { "id": 3, "name": "bob", "age": 25 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"2");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 2,
-            number_of_documents: 2,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":2,"doggo":{"name":"jean","age":20}}
-        {"id":3,"name":"bob","age":25}
-        "###);
-    }
-
-    #[test]
-    fn delete_the_same_document_multiple_time() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let (builder, removed) =
-            builder.remove_documents(vec![S("1"), S("2"), S("1"), S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"0");
-
-        let documents = documents!([
-            { "id": 1, "doggo": "kevin" },
-            { "id": 2, "doggo": { "name": "jean", "age": 20 } },
-            { "id": 3, "name": "bob", "age": 25 },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"3");
-
-        let (builder, removed) =
-            builder.remove_documents(vec![S("1"), S("2"), S("1"), S("2")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"2");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 3,
-            number_of_documents: 1,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":3,"name":"bob","age":25}
-        "###);
-    }
-
-    #[test]
-    fn add_document_and_in_another_transform_delete_the_document_then_add_it_again() {
-        let mut index = TempIndex::new();
-        index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments;
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let documents = documents!([
-            { "id": 1, "doggo": "kevin" },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"1");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 1,
-            number_of_documents: 1,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":1,"doggo":"kevin"}
-        "###);
-
-        // A first batch of documents has been inserted
-
-        let mut wtxn = index.write_txn().unwrap();
-        let builder = IndexDocuments::new(
-            &mut wtxn,
-            &index,
-            &index.indexer_config,
-            index.index_documents_config.clone(),
-            |_| (),
-            || false,
-        )
-        .unwrap();
-
-        let (builder, removed) = builder.remove_documents(vec![S("1")]).unwrap();
-        insta::assert_display_snapshot!(removed.unwrap(), @"1");
-
-        let documents = documents!([
-            { "id": 1, "catto": "jorts" },
-        ]);
-        let (builder, added) = builder.add_documents(documents).unwrap();
-        insta::assert_display_snapshot!(added.unwrap(), @"1");
-
-        let addition = builder.execute().unwrap();
-        insta::assert_debug_snapshot!(addition, @r###"
-        DocumentAdditionResult {
-            indexed_documents: 1,
-            number_of_documents: 1,
-        }
-        "###);
-        wtxn.commit().unwrap();
-
-        db_snap!(index, documents, @r###"
-        {"id":1,"catto":"jorts"}
-        "###);
-    }
 }

milli/src/update/index_documents/transform.rs

@@ -12,9 +12,7 @@ use roaring::RoaringBitmap;
 use serde_json::Value;
 use smartstring::SmartString;
 
-use super::helpers::{
-    create_sorter, create_writer, keep_latest_obkv, merge_obkvs_and_operations, MergeFn,
-};
+use super::helpers::{create_sorter, create_writer, keep_latest_obkv, merge_obkvs, MergeFn};
 use super::{IndexDocumentsMethod, IndexerConfig};
 use crate::documents::{DocumentsBatchIndex, EnrichedDocument, EnrichedDocumentsBatchReader};
 use crate::error::{Error, InternalError, UserError};
@@ -52,12 +50,8 @@ pub struct Transform<'a, 'i> {
     pub index_documents_method: IndexDocumentsMethod,
     available_documents_ids: AvailableDocumentsIds,
 
-    // Both grenad follows the same format:
-    // key | value
-    // u32 | 1 byte for the Operation byte, the rest is the obkv of the document stored
     original_sorter: grenad::Sorter<MergeFn>,
     flattened_sorter: grenad::Sorter<MergeFn>,
-
     replaced_documents_ids: RoaringBitmap,
     new_documents_ids: RoaringBitmap,
     // To increase the cache locality and decrease the heap usage we use compact smartstring.
@@ -65,14 +59,6 @@ pub struct Transform<'a, 'i> {
     documents_count: usize,
 }
 
-/// This enum is specific to the grenad sorter stored in the transform.
-/// It's used as the first byte of the grenads and tells you if the document id was an addition or a deletion.
-#[repr(u8)]
-pub enum Operation {
-    Addition,
-    Deletion,
-}
-
 /// Create a mapping between the field ids found in the document batch and the one that were
 /// already present in the index.
 ///
@@ -108,7 +94,7 @@ impl<'a, 'i> Transform<'a, 'i> {
         // with the same user id must be merged or fully replaced in the same batch.
         let merge_function = match index_documents_method {
             IndexDocumentsMethod::ReplaceDocuments => keep_latest_obkv,
-            IndexDocumentsMethod::UpdateDocuments => merge_obkvs_and_operations,
+            IndexDocumentsMethod::UpdateDocuments => merge_obkvs,
         };
 
         // We initialize the sorter with the user indexing settings.
@@ -165,7 +151,9 @@ impl<'a, 'i> Transform<'a, 'i> {
         FA: Fn() -> bool + Sync,
     {
         let (mut cursor, fields_index) = reader.into_cursor_and_fields_index();
+
         let external_documents_ids = self.index.external_documents_ids(wtxn)?;
+
         let mapping = create_fields_mapping(&mut self.fields_ids_map, &fields_index)?;
 
         let primary_key = cursor.primary_key().to_string();
@@ -173,7 +161,6 @@ impl<'a, 'i> Transform<'a, 'i> {
             self.fields_ids_map.insert(&primary_key).ok_or(UserError::AttributeLimitReached)?;
 
         let mut obkv_buffer = Vec::new();
-        let mut document_sorter_buffer = Vec::new();
         let mut documents_count = 0;
         let mut docid_buffer: Vec<u8> = Vec::new();
         let mut field_buffer: Vec<(u16, Cow<[u8]>)> = Vec::new();
@@ -225,13 +212,10 @@ impl<'a, 'i> Transform<'a, 'i> {
                 Entry::Occupied(entry) => *entry.get() as u32,
                 Entry::Vacant(entry) => {
                     // If the document was already in the db we mark it as a replaced document.
-                    // It'll be deleted later.
+                    // It'll be deleted later. We keep its original docid to insert it in the grenad.
                     if let Some(docid) = external_documents_ids.get(entry.key()) {
-                        // If it was already in the list of replaced documents it means it was deleted
-                        // by the remove_document method. We should starts as if it never existed.
-                        if self.replaced_documents_ids.insert(docid) {
-                            original_docid = Some(docid);
-                        }
+                        self.replaced_documents_ids.insert(docid);
+                        original_docid = Some(docid);
                     }
                     let docid = self
                         .available_documents_ids
@@ -264,46 +248,26 @@ impl<'a, 'i> Transform<'a, 'i> {
                     skip_insertion = true;
                 } else {
                     // we associate the base document with the new key, everything will get merged later.
-                    document_sorter_buffer.clear();
-                    document_sorter_buffer.push(Operation::Addition as u8);
-                    document_sorter_buffer.extend_from_slice(base_obkv);
-                    self.original_sorter.insert(docid.to_be_bytes(), &document_sorter_buffer)?;
+                    self.original_sorter.insert(docid.to_be_bytes(), base_obkv)?;
                     match self.flatten_from_fields_ids_map(KvReader::new(base_obkv))? {
-                        Some(flattened_obkv) => {
-                            // we recreate our buffer with the flattened documents
-                            document_sorter_buffer.clear();
-                            document_sorter_buffer.push(Operation::Addition as u8);
-                            document_sorter_buffer.extend_from_slice(&flattened_obkv);
-                            self.flattened_sorter
-                                .insert(docid.to_be_bytes(), &document_sorter_buffer)?
+                        Some(buffer) => {
+                            self.flattened_sorter.insert(docid.to_be_bytes(), &buffer)?
                         }
-                        None => self
-                            .flattened_sorter
-                            .insert(docid.to_be_bytes(), &document_sorter_buffer)?,
+                        None => self.flattened_sorter.insert(docid.to_be_bytes(), base_obkv)?,
                     }
                 }
             }
 
             if !skip_insertion {
                 self.new_documents_ids.insert(docid);
-
-                document_sorter_buffer.clear();
-                document_sorter_buffer.push(Operation::Addition as u8);
-                document_sorter_buffer.extend_from_slice(&obkv_buffer);
                 // We use the extracted/generated user id as the key for this document.
-                self.original_sorter.insert(docid.to_be_bytes(), &document_sorter_buffer)?;
+                self.original_sorter.insert(docid.to_be_bytes(), obkv_buffer.clone())?;
 
                 match self.flatten_from_fields_ids_map(KvReader::new(&obkv_buffer))? {
-                    Some(flattened_obkv) => {
-                        document_sorter_buffer.clear();
-                        document_sorter_buffer.push(Operation::Addition as u8);
-                        document_sorter_buffer.extend_from_slice(&flattened_obkv);
-                        self.flattened_sorter
-                            .insert(docid.to_be_bytes(), &document_sorter_buffer)?
+                    Some(buffer) => self.flattened_sorter.insert(docid.to_be_bytes(), &buffer)?,
+                    None => {
+                        self.flattened_sorter.insert(docid.to_be_bytes(), obkv_buffer.clone())?
                     }
-                    None => self
-                        .flattened_sorter
-                        .insert(docid.to_be_bytes(), &document_sorter_buffer)?,
                 }
             }
             documents_count += 1;
@@ -329,73 +293,6 @@ impl<'a, 'i> Transform<'a, 'i> {
         Ok(documents_count)
     }
 
-    /// The counter part of `read_documents` that removes documents either from the transform or the database.
-    /// It can be called before, after or in between two calls of the `read_documents`.
-    ///
-    /// It needs to update all the internal datastructure in the transform.
-    /// - If the document is coming from the database -> it's marked as a to_delete document
-    /// - If the document to remove was inserted by the `read_documents` method before AND was present in the db,
-    ///   it's marked as `to_delete` + added into the grenad to ensure we don't reinsert it.
-    /// - If the document to remove was inserted by the `read_documents` method before but was NOT present in the db,
-    ///   it's added into the grenad to ensure we don't insert it + removed from the list of new documents ids.
-    /// - If the document to remove was not present in either the db or the transform we do nothing.
-    pub fn remove_documents<FA>(
-        &mut self,
-        mut to_remove: Vec<String>,
-        wtxn: &mut heed::RwTxn,
-        should_abort: FA,
-    ) -> Result<usize>
-    where
-        FA: Fn() -> bool + Sync,
-    {
-        // there may be duplicates in the documents to remove.
-        to_remove.sort_unstable();
-        to_remove.dedup();
-
-        let external_documents_ids = self.index.external_documents_ids(wtxn)?;
-
-        let mut documents_deleted = 0;
-        for to_remove in to_remove {
-            if should_abort() {
-                return Err(Error::InternalError(InternalError::AbortedIndexation));
-            }
-
-            match self.new_external_documents_ids_builder.entry((*to_remove).into()) {
-                // if the document was added in a previous iteration of the transform we make it as deleted in the sorters.
-                Entry::Occupied(entry) => {
-                    let doc_id = *entry.get() as u32;
-                    self.original_sorter
-                        .insert(doc_id.to_be_bytes(), [Operation::Deletion as u8])?;
-                    self.flattened_sorter
-                        .insert(doc_id.to_be_bytes(), [Operation::Deletion as u8])?;
-
-                    // we must NOT update the list of replaced_documents_ids
-                    // Either:
-                    // 1. It's already in it and there is nothing to do
-                    // 2. It wasn't in it because the document was created by a previous batch and since
-                    //    we're removing it there is nothing to do.
-                    self.new_documents_ids.remove(doc_id);
-                    entry.remove_entry();
-                }
-                Entry::Vacant(entry) => {
-                    // If the document was already in the db we mark it as a `to_delete` document.
-                    // It'll be deleted later. We don't need to push anything to the sorters.
-                    if let Some(docid) = external_documents_ids.get(entry.key()) {
-                        self.replaced_documents_ids.insert(docid);
-                    } else {
-                        // if the document is nowehere to be found, there is nothing to do and we must NOT
-                        // increment the count of documents_deleted
-                        continue;
-                    }
-                }
-            };
-
-            documents_deleted += 1;
-        }
-
-        Ok(documents_deleted)
-    }
-
     // Flatten a document from the fields ids map contained in self and insert the new
     // created fields. Returns `None` if the document doesn't need to be flattened.
     fn flatten_from_fields_ids_map(&mut self, obkv: KvReader<FieldId>) -> Result<Option<Vec<u8>>> {
@@ -590,11 +487,6 @@ impl<'a, 'i> Transform<'a, 'i> {
         let mut documents_count = 0;
 
         while let Some((key, val)) = iter.next()? {
-            if val[0] == Operation::Deletion as u8 {
-                continue;
-            }
-            let val = &val[1..];
-
             // send a callback to show at which step we are
             documents_count += 1;
             progress_callback(UpdateIndexingStep::ComputeIdsAndMergeDocuments {
@@ -626,18 +518,9 @@ impl<'a, 'i> Transform<'a, 'i> {
             self.indexer_settings.chunk_compression_level,
             tempfile::tempfile()?,
         );
-
-        // Once we have written all the documents into the final sorter, we write the nested documents
-        // into this writer.
-        // We get rids of the `Operation` byte and skip the deleted documents as well.
-        let mut iter = self.flattened_sorter.into_stream_merger_iter()?;
-        while let Some((key, val)) = iter.next()? {
-            if val[0] == Operation::Deletion as u8 {
-                continue;
-            }
-            let val = &val[1..];
-            writer.insert(key, val)?;
-        }
+        // Once we have written all the documents into the final sorter, we write the documents
+        // into this writer, extract the file and reset the seek to be able to read it again.
+        self.flattened_sorter.write_into_stream_writer(&mut writer)?;
         let mut flattened_documents = writer.into_inner()?;
         flattened_documents.rewind()?;
 
@@ -818,45 +701,3 @@ impl TransformOutput {
             .collect())
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn merge_obkvs() {
-        let mut doc_0 = Vec::new();
-        let mut kv_writer = KvWriter::new(&mut doc_0);
-        kv_writer.insert(0_u8, [0]).unwrap();
-        kv_writer.finish().unwrap();
-        doc_0.insert(0, Operation::Addition as u8);
-
-        let ret = merge_obkvs_and_operations(&[], &[Cow::from(doc_0.as_slice())]).unwrap();
-        assert_eq!(*ret, doc_0);
-
-        let ret = merge_obkvs_and_operations(
-            &[],
-            &[Cow::from([Operation::Deletion as u8].as_slice()), Cow::from(doc_0.as_slice())],
-        )
-        .unwrap();
-        assert_eq!(*ret, doc_0);
-
-        let ret = merge_obkvs_and_operations(
-            &[],
-            &[Cow::from(doc_0.as_slice()), Cow::from([Operation::Deletion as u8].as_slice())],
-        )
-        .unwrap();
-        assert_eq!(*ret, [Operation::Deletion as u8]);
-
-        let ret = merge_obkvs_and_operations(
-            &[],
-            &[
-                Cow::from([Operation::Addition as u8, 1].as_slice()),
-                Cow::from([Operation::Deletion as u8].as_slice()),
-                Cow::from(doc_0.as_slice()),
-            ],
-        )
-        .unwrap();
-        assert_eq!(*ret, doc_0);
-    }
-}

milli/src/update/indexer_config.rs

@@ -11,6 +11,7 @@ pub struct IndexerConfig {
     pub chunk_compression_level: Option<u32>,
     pub thread_pool: Option<ThreadPool>,
     pub max_positions_per_attributes: Option<u32>,
+    pub skip_index_budget: bool,
 }
 
 impl Default for IndexerConfig {
@@ -24,6 +25,7 @@ impl Default for IndexerConfig {
             chunk_compression_level: None,
             thread_pool: None,
             max_positions_per_attributes: None,
+            skip_index_budget: false,
         }
     }
 }

permissive-json-pointer/Cargo.toml

@@ -1,16 +1,9 @@
 [package]
 name = "permissive-json-pointer"
+version = "1.0.0"
+edition = "2021"
 description = "A permissive json pointer"
 readme = "README.md"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-# description.workspace = true
-homepage.workspace = true
-# readme.workspace = true
-edition.workspace = true
-license.workspace = true
 
 [dependencies]
 serde_json = "1.0"