Only provide the last batch info

remove-me: Introduce the MEILI_MAX_INDEX_SIZE env variable to limit the size of an index
Fix some errors
2025-11-25 05:57:11 +00:00 · 2025-11-20 12:03:11 +01:00 · 2025-11-20 11:47:59 +01:00 · 2025-11-20 11:47:59 +01:00 · 2025-11-20 11:47:58 +01:00 · 2025-11-20 11:47:58 +01:00
40 changed files with 370 additions and 184 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -584,7 +584,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2

 [[package]]
 name = "benchmarks"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "bumpalo",
@@ -794,7 +794,7 @@ dependencies = [

 [[package]]
 name = "build-info"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "time",
@@ -1784,7 +1784,7 @@ dependencies = [

 [[package]]
 name = "dump"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "big_s",
@@ -2027,7 +2027,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"

 [[package]]
 name = "file-store"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "tempfile",
 "thiserror 2.0.16",
@@ -2049,7 +2049,7 @@ dependencies = [

 [[package]]
 name = "filter-parser"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "insta",
 "levenshtein_automata",
@@ -2077,7 +2077,7 @@ dependencies = [

 [[package]]
 name = "flatten-serde-json"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "criterion",
 "serde_json",
@@ -2234,7 +2234,7 @@ dependencies = [

 [[package]]
 name = "fuzzers"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "arbitrary",
 "bumpalo",
@@ -2922,6 +2922,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"

+[[package]]
+name = "humantime"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+
 [[package]]
 name = "hyper"
 version = "1.7.0"
@@ -3188,7 +3194,7 @@ dependencies = [

 [[package]]
 name = "index-scheduler"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "backoff",
@@ -3461,7 +3467,7 @@ dependencies = [

 [[package]]
 name = "json-depth-checker"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "criterion",
 "serde_json",
@@ -3980,7 +3986,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"

 [[package]]
 name = "meili-snap"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "insta",
 "md5",
@@ -3991,7 +3997,7 @@ dependencies = [

 [[package]]
 name = "meilisearch"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "actix-cors",
 "actix-http",
@@ -4020,6 +4026,7 @@ dependencies = [
 "futures",
 "futures-util",
 "hex",
+ "humantime",
 "index-scheduler",
 "indexmap",
 "insta",
@@ -4088,7 +4095,7 @@ dependencies = [

 [[package]]
 name = "meilisearch-auth"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "base64 0.22.1",
 "enum-iterator",
@@ -4107,7 +4114,7 @@ dependencies = [

 [[package]]
 name = "meilisearch-types"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "actix-web",
 "anyhow",
@@ -4142,7 +4149,7 @@ dependencies = [

 [[package]]
 name = "meilitool"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "clap",
@@ -4176,7 +4183,7 @@ dependencies = [

 [[package]]
 name = "milli"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "arroy",
 "bbqueue",
@@ -4757,7 +4764,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"

 [[package]]
 name = "permissive-json-pointer"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "big_s",
 "serde_json",
@@ -7879,7 +7886,7 @@ dependencies = [

 [[package]]
 name = "xtask"
-version = "1.27.0"
+version = "1.26.0"
 dependencies = [
 "anyhow",
 "build-info",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -23,7 +23,7 @@ members = [
 ]

 [workspace.package]
-version = "1.27.0"
+version = "1.26.0"
 authors = [
    "Quentin de Quelen <quentin@dequelen.me>",
    "Clément Renault <clement@meilisearch.com>",
--- a/crates/dump/src/lib.rs
+++ b/crates/dump/src/lib.rs
@@ -341,6 +341,7 @@ pub(crate) mod test {
            prefix_search: Setting::NotSet,
            chat: Setting::NotSet,
            vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
            _kind: std::marker::PhantomData,
        };
        settings.check()
--- a/crates/dump/src/reader/compat/v5_to_v6.rs
+++ b/crates/dump/src/reader/compat/v5_to_v6.rs
@@ -423,6 +423,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
            prefix_search: v6::Setting::NotSet,
            chat: v6::Setting::NotSet,
            vector_store: v6::Setting::NotSet,
+            execute_after_update: v6::Setting::NotSet,
            _kind: std::marker::PhantomData,
        }
    }
--- a/crates/index-scheduler/src/index_mapper/index_map.rs
+++ b/crates/index-scheduler/src/index_mapper/index_map.rs
@@ -306,6 +306,18 @@ fn create_or_open_index(
 ) -> Result<Index> {
    let options = EnvOpenOptions::new();
    let mut options = options.read_txn_without_tls();
+
+    let map_size = match std::env::var("MEILI_MAX_INDEX_SIZE") {
+        Ok(max_size) => {
+            let max_size = max_size.parse().unwrap();
+            map_size.min(max_size)
+        }
+        Err(VarError::NotPresent) => map_size,
+        Err(VarError::NotUnicode(e)) => {
+            panic!("Non unicode max index size in `MEILI_MAX_INDEX_SIZE`: {e:?}")
+        }
+    };
+
    options.map_size(clamp_to_page_size(map_size));

    // You can find more details about this experimental
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
 [timestamp] [4,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.27.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.26.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
@@ -37,7 +37,7 @@ catto [1,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.27.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.26.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.27.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.26.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
--- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap
+++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap
@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 27, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 26, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.27.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.26.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]
--- a/crates/index-scheduler/src/upgrade/mod.rs
+++ b/crates/index-scheduler/src/upgrade/mod.rs
@@ -50,7 +50,6 @@ pub fn upgrade_index_scheduler(
        (1, 24, _) => 0,
        (1, 25, _) => 0,
        (1, 26, _) => 0,
-        (1, 27, _) => 0,
        (major, minor, patch) => {
            if major > current_major
                || (major == current_major && minor > current_minor)
--- a/crates/meilisearch-types/src/error.rs
+++ b/crates/meilisearch-types/src/error.rs
@@ -324,6 +324,7 @@ InvalidSettingsDisplayedAttributes             , InvalidRequest       , BAD_REQU
 InvalidSettingsDistinctAttribute               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsProximityPrecision              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFacetSearch                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsexecuteAfterUpdate              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsPrefixSearch                    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFaceting                        , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFilterableAttributes            , InvalidRequest       , BAD_REQUEST ;
--- a/crates/meilisearch-types/src/settings.rs
+++ b/crates/meilisearch-types/src/settings.rs
@@ -326,6 +326,12 @@ pub struct Settings<T> {
    #[schema(value_type = Option<VectorStoreBackend>)]
    pub vector_store: Setting<VectorStoreBackend>,

+    /// Function to execute after an update
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsexecuteAfterUpdate>)]
+    #[schema(value_type = Option<String>, example = json!("doc.likes += 1"))]
+    pub execute_after_update: Setting<String>,
+
    #[serde(skip)]
    #[deserr(skip)]
    pub _kind: PhantomData<T>,
@@ -395,6 +401,7 @@ impl Settings<Checked> {
            prefix_search: Setting::Reset,
            chat: Setting::Reset,
            vector_store: Setting::Reset,
+            execute_after_update: Setting::Reset,
            _kind: PhantomData,
        }
    }
@@ -423,6 +430,7 @@ impl Settings<Checked> {
            prefix_search,
            chat,
            vector_store,
+            execute_after_update,
            _kind,
        } = self;

@@ -449,6 +457,7 @@ impl Settings<Checked> {
            prefix_search,
            vector_store,
            chat,
+            execute_after_update,
            _kind: PhantomData,
        }
    }
@@ -501,6 +510,7 @@ impl Settings<Unchecked> {
            prefix_search: self.prefix_search,
            chat: self.chat,
            vector_store: self.vector_store,
+            execute_after_update: self.execute_after_update,
            _kind: PhantomData,
        }
    }
@@ -582,6 +592,10 @@ impl Settings<Unchecked> {
            prefix_search: other.prefix_search.or(self.prefix_search),
            chat: other.chat.clone().or(self.chat.clone()),
            vector_store: other.vector_store.or(self.vector_store),
+            execute_after_update: other
+                .execute_after_update
+                .clone()
+                .or(self.execute_after_update.clone()),
            _kind: PhantomData,
        }
    }
@@ -622,6 +636,7 @@ pub fn apply_settings_to_builder(
        prefix_search,
        chat,
        vector_store,
+        execute_after_update,
        _kind,
    } = settings;

@@ -845,6 +860,14 @@ pub fn apply_settings_to_builder(
        Setting::Reset => builder.reset_vector_store(),
        Setting::NotSet => (),
    }
+
+    match execute_after_update {
+        Setting::Set(execute_after_update) => {
+            builder.set_execute_after_update(execute_after_update.clone())
+        }
+        Setting::Reset => builder.reset_execute_after_update(),
+        Setting::NotSet => (),
+    }
 }

 pub enum SecretPolicy {
@@ -944,13 +967,13 @@ pub fn settings(
        .collect();

    let vector_store = index.get_vector_store(rtxn)?;
-
    let embedders = Setting::Set(embedders);
    let search_cutoff_ms = index.search_cutoff(rtxn)?;
    let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
    let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
    let facet_search = index.facet_search(rtxn)?;
    let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
+    let execute_after_update = index.execute_after_update(rtxn)?;

    let mut settings = Settings {
        displayed_attributes: match displayed_attributes {
@@ -995,6 +1018,10 @@ pub fn settings(
            Some(vector_store) => Setting::Set(vector_store),
            None => Setting::Reset,
        },
+        execute_after_update: match execute_after_update {
+            Some(function) => Setting::Set(function.to_string()),
+            None => Setting::NotSet,
+        },
        _kind: PhantomData,
    };

@@ -1225,6 +1252,7 @@ pub(crate) mod test {
            prefix_search: Setting::NotSet,
            chat: Setting::NotSet,
            vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
            _kind: PhantomData::<Unchecked>,
        };

@@ -1258,7 +1286,7 @@ pub(crate) mod test {
            prefix_search: Setting::NotSet,
            chat: Setting::NotSet,
            vector_store: Setting::NotSet,
-
+            execute_after_update: Setting::NotSet,
            _kind: PhantomData::<Unchecked>,
        };

--- a/crates/meilisearch/Cargo.toml
+++ b/crates/meilisearch/Cargo.toml
@@ -117,7 +117,7 @@ secrecy = "0.10.3"
 actix-web-lab = { version = "0.24.1", default-features = false }
 urlencoding = "2.1.3"
 backoff = { version = "0.4.0", features = ["tokio"] }
-
+humantime = { version = "2.3.0", default-features = false }

 [dev-dependencies]
 actix-rt = "2.10.0"
--- a/crates/meilisearch/src/lib.rs
+++ b/crates/meilisearch/src/lib.rs
@@ -231,14 +231,8 @@ pub fn setup_meilisearch(
        max_number_of_tasks: 1_000_000,
        max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks,
        batched_tasks_size_limit: opt.experimental_limit_batched_tasks_total_size.map_or_else(
-            || {
-                opt.indexer_options
-                    .max_indexing_memory
-                    // By default, we use half of the available memory to determine the size of batched tasks
-                    .map_or(u64::MAX, |mem| mem.as_u64() / 2)
-                    // And never exceed 10 GiB when we infer the limit
-                    .min(10 * 1024 * 1024 * 1024)
-            },
+            // By default, we use half of the available memory to determine the size of batched tasks
+            || opt.indexer_options.max_indexing_memory.map_or(u64::MAX, |mem| mem.as_u64() / 2),
            |size| size.as_u64(),
        ),
        index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize,
--- a/crates/meilisearch/src/metrics.rs
+++ b/crates/meilisearch/src/metrics.rs
@@ -1,7 +1,8 @@
 use lazy_static::lazy_static;
 use prometheus::{
-    opts, register_gauge, register_histogram_vec, register_int_counter_vec, register_int_gauge,
-    register_int_gauge_vec, Gauge, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
+    opts, register_gauge, register_gauge_vec, register_histogram_vec, register_int_counter_vec,
+    register_int_gauge, register_int_gauge_vec, Gauge, GaugeVec, HistogramVec, IntCounterVec,
+    IntGauge, IntGaugeVec,
 };

 lazy_static! {
@@ -73,6 +74,20 @@ lazy_static! {
        &["kind", "value"]
    )
    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE: GaugeVec = register_gauge_vec!(
+        opts!("meilisearch_batch_running_progress_trace", "The currently running progress trace"),
+        &["batch_uid", "step_name"]
+    )
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS: IntGaugeVec =
+        register_int_gauge_vec!(
+            opts!(
+                "meilisearch_last_finished_batches_progress_trace_ms",
+                "The last few batches progress trace in milliseconds"
+            ),
+            &["batch_uid", "step_name"]
+        )
+        .expect("Can't create a metric");
    pub static ref MEILISEARCH_LAST_UPDATE: IntGauge =
        register_int_gauge!(opts!("meilisearch_last_update", "Meilisearch Last Update"))
            .expect("Can't create a metric");
--- a/crates/meilisearch/src/option.rs
+++ b/crates/meilisearch/src/option.rs
@@ -474,8 +474,7 @@ pub struct Opt {
    pub experimental_max_number_of_batched_tasks: usize,

    /// Experimentally controls the maximum total size, in bytes, of tasks that will be processed
-    /// simultaneously. When unspecified, defaults to half of the maximum indexing memory and
-    /// clamped to 10 GiB.
+    /// simultaneously. When unspecified, defaults to half of the maximum indexing memory.
    ///
    /// See: <https://github.com/orgs/meilisearch/discussions/801>
    #[clap(long, env = MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE)]
--- a/crates/meilisearch/src/routes/indexes/settings.rs
+++ b/crates/meilisearch/src/routes/indexes/settings.rs
@@ -498,6 +498,17 @@ make_setting_routes!(
        camelcase_attr: "facetSearch",
        analytics: FacetSearchAnalytics
    },
+    {
+        route: "/execute-after-update",
+        update_verb: put,
+        value_type: String,
+        err_type: meilisearch_types::deserr::DeserrJsonError<
+            meilisearch_types::error::deserr_codes::InvalidSettingsexecuteAfterUpdate,
+        >,
+        attr: execute_after_update,
+        camelcase_attr: "executeAfterUpdate",
+        analytics: ExecuteAfterUpdateAnalytics
+    },
    {
        route: "/prefix-search",
        update_verb: put,
@@ -619,6 +630,9 @@ pub async fn update_all(
                new_settings.non_separator_tokens.as_ref().set(),
            ),
            facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
+            execute_after_update: ExecuteAfterUpdateAnalytics::new(
+                new_settings.execute_after_update.as_ref().set(),
+            ),
            prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
            chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
            vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),
--- a/crates/meilisearch/src/routes/indexes/settings_analytics.rs
+++ b/crates/meilisearch/src/routes/indexes/settings_analytics.rs
@@ -42,6 +42,7 @@ pub struct SettingsAnalytics {
    pub prefix_search: PrefixSearchAnalytics,
    pub chat: ChatAnalytics,
    pub vector_store: VectorStoreAnalytics,
+    pub execute_after_update: ExecuteAfterUpdateAnalytics,
 }

 impl Aggregate for SettingsAnalytics {
@@ -197,6 +198,9 @@ impl Aggregate for SettingsAnalytics {
                set: new.facet_search.set | self.facet_search.set,
                value: new.facet_search.value.or(self.facet_search.value),
            },
+            execute_after_update: ExecuteAfterUpdateAnalytics {
+                set: new.execute_after_update.set | self.execute_after_update.set,
+            },
            prefix_search: PrefixSearchAnalytics {
                set: new.prefix_search.set | self.prefix_search.set,
                value: new.prefix_search.value.or(self.prefix_search.value),
@@ -669,6 +673,21 @@ impl FacetSearchAnalytics {
    }
 }

+#[derive(Serialize, Default)]
+pub struct ExecuteAfterUpdateAnalytics {
+    pub set: bool,
+}
+
+impl ExecuteAfterUpdateAnalytics {
+    pub fn new(distinct: Option<&String>) -> Self {
+        Self { set: distinct.is_some() }
+    }
+
+    pub fn into_settings(self) -> SettingsAnalytics {
+        SettingsAnalytics { execute_after_update: self, ..Default::default() }
+    }
+}
+
 #[derive(Serialize, Default)]
 pub struct PrefixSearchAnalytics {
    pub set: bool,
--- a/crates/meilisearch/src/routes/metrics.rs
+++ b/crates/meilisearch/src/routes/metrics.rs
@@ -4,6 +4,7 @@ use index_scheduler::{IndexScheduler, Query};
 use meilisearch_auth::AuthController;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::actions;
+use meilisearch_types::milli::progress::ProgressStepView;
 use meilisearch_types::tasks::Status;
 use prometheus::{Encoder, TextEncoder};
 use time::OffsetDateTime;
@@ -38,6 +39,12 @@ pub fn configure(config: &mut web::ServiceConfig) {
 # HELP meilisearch_db_size_bytes Meilisearch DB Size In Bytes
 # TYPE meilisearch_db_size_bytes gauge
 meilisearch_db_size_bytes 1130496
+# HELP meilisearch_batch_running_progress_trace The currently running progress trace
+# TYPE meilisearch_batch_running_progress_trace gauge
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="document"} 0.710618582519409
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="extracting word proximity"} 0.2222222222222222
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="indexing"} 0.6666666666666666
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="processing tasks"} 0
 # HELP meilisearch_http_requests_total Meilisearch HTTP requests total
 # TYPE meilisearch_http_requests_total counter
 meilisearch_http_requests_total{method="GET",path="/metrics",status="400"} 1
@@ -61,6 +68,13 @@ meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="1
 meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="+Inf"} 0
 meilisearch_http_response_time_seconds_sum{method="GET",path="/metrics"} 0
 meilisearch_http_response_time_seconds_count{method="GET",path="/metrics"} 0
+# HELP meilisearch_last_finished_batches_progress_trace_ms The last few batches progress trace in milliseconds
+# TYPE meilisearch_last_finished_batches_progress_trace_ms gauge
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks"} 19360
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes"} 368
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes > preparing payloads"} 367
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes > preparing payloads > payload"} 367
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > indexing"} 18970
 # HELP meilisearch_index_count Meilisearch Index Count
 # TYPE meilisearch_index_count gauge
 meilisearch_index_count 1
@@ -148,6 +162,46 @@ pub async fn get_metrics(
        }
    }

+    // Fetch and expose the current progressing step
+    crate::metrics::MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE.reset();
+    let (batches, _total) = index_scheduler.get_batches_from_authorized_indexes(
+        &Query { statuses: Some(vec![Status::Processing]), ..Query::default() },
+        auth_filters,
+    )?;
+    if let Some(batch) = batches.into_iter().next() {
+        let batch_uid = batch.uid.to_string();
+        if let Some(progress) = batch.progress {
+            for ProgressStepView { current_step, finished, total } in progress.steps {
+                crate::metrics::MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE
+                    .with_label_values(&[batch_uid.as_str(), current_step.as_ref()])
+                    // We return the completion ratio of the current step
+                    .set(finished as f64 / total as f64);
+            }
+        }
+    }
+
+    crate::metrics::MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS.reset();
+    let (batches, _total) = index_scheduler.get_batches_from_authorized_indexes(
+        // Fetch the finished batches...
+        &Query { statuses: Some(vec![Status::Succeeded, Status::Failed]), ..Query::default() },
+        auth_filters,
+    )?;
+    // ...and get the last batch only.
+    if let Some(batch) = batches.into_iter().next() {
+        let batch_uid = batch.uid.to_string();
+        for (step_name, duration_str) in batch.stats.progress_trace {
+            let Some(duration_str) = duration_str.as_str() else { continue };
+            match humantime::parse_duration(duration_str) {
+                Ok(duration) => {
+                    crate::metrics::MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS
+                        .with_label_values(&[&batch_uid, &step_name])
+                        .set(duration.as_millis() as i64);
+                }
+                Err(e) => tracing::error!("Failed to parse duration: {e}"),
+            }
+        }
+    }
+
    if let Some(last_update) = response.last_update {
        crate::metrics::MEILISEARCH_LAST_UPDATE.set(last_update.unix_timestamp());
    }
--- a/crates/meilisearch/tests/documents/get_documents.rs
+++ b/crates/meilisearch/tests/documents/get_documents.rs
@@ -1339,117 +1339,3 @@ async fn get_document_with_vectors() {
    }
    "###);
 }
-
-#[actix_rt::test]
-async fn test_fetch_documents_pagination_with_sorting() {
-    let server = Server::new_shared();
-    let index = server.unique_index();
-    let (task, _code) = index.create(None).await;
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Set name as sortable attribute
-    let (task, code) = index.update_settings_sortable_attributes(json!(["name"])).await;
-    assert_eq!(code, 202);
-    server.wait_task(task.uid()).await.succeeded();
-
-    let documents = json!((0..50)
-        .map(|i| json!({"id": i, "name": format!("doc_{:05}", std::cmp::min(i, 5))}))
-        .collect::<Vec<_>>());
-
-    // Add documents as described in the bug report
-    let (task, code) = index.add_documents(documents, None).await;
-    assert_eq!(code, 202);
-    server.wait_task(task.uid()).await.succeeded();
-
-    // Request 1 (first page): offset 0, limit 2
-    let (response, code) = index
-        .fetch_documents(json!({
-            "offset": 0,
-            "limit": 2,
-            "sort": ["name:asc"]
-        }))
-        .await;
-    assert_eq!(code, 200);
-    let results = response["results"].as_array().unwrap();
-    snapshot!(json_string!(results), @r###"
-    [
-      {
-        "id": 0,
-        "name": "doc_00000"
-      },
-      {
-        "id": 1,
-        "name": "doc_00001"
-      }
-    ]
-    "###);
-
-    // Request 2 (second page): offset 2, limit 2
-    let (response, code) = index
-        .fetch_documents(json!({
-            "offset": 2,
-            "limit": 2,
-            "sort": ["name:asc"]
-        }))
-        .await;
-    assert_eq!(code, 200);
-    let results = response["results"].as_array().unwrap();
-    snapshot!(json_string!(results), @r###"
-    [
-      {
-        "id": 2,
-        "name": "doc_00002"
-      },
-      {
-        "id": 3,
-        "name": "doc_00003"
-      }
-    ]
-    "###);
-
-    // Request 3 (third page): offset 4, limit 2
-    let (response, code) = index
-        .fetch_documents(json!({
-            "offset": 4,
-            "limit": 2,
-            "sort": ["name:asc"]
-        }))
-        .await;
-    assert_eq!(code, 200);
-    let results = response["results"].as_array().unwrap();
-    snapshot!(json_string!(results), @r###"
-    [
-      {
-        "id": 4,
-        "name": "doc_00004"
-      },
-      {
-        "id": 5,
-        "name": "doc_00005"
-      }
-    ]
-    "###);
-
-    // Request 4 (fourth page): offset 6, limit 2
-    let (response, code) = index
-        .fetch_documents(json!({
-            "offset": 6,
-            "limit": 2,
-            "sort": ["name:asc"]
-        }))
-        .await;
-    assert_eq!(code, 200);
-    let results = response["results"].as_array().unwrap();
-    snapshot!(json_string!(results), @r###"
-    [
-      {
-        "id": 6,
-        "name": "doc_00005"
-      },
-      {
-        "id": 7,
-        "name": "doc_00005"
-      }
-    ]
-    "###);
-}
--- a/crates/meilisearch/tests/upgrade/mod.rs
+++ b/crates/meilisearch/tests/upgrade/mod.rs
@@ -43,7 +43,7 @@ async fn version_too_old() {
    std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
    let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
    let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.27.0");
+    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.26.0");
 }

 #[actix_rt::test]
@@ -58,7 +58,7 @@ async fn version_requires_downgrade() {
    std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
    let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
    let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.27.1 is higher than the Meilisearch version 1.27.0. Downgrade is not supported");
+    snapshot!(err, @"Database version 1.26.1 is higher than the Meilisearch version 1.26.0. Downgrade is not supported");
 }

 #[actix_rt::test]
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "progress": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "stats": {
        "totalNbTasks": 1,
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "progress": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "stats": {
        "totalNbTasks": 1,
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "progress": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "stats": {
        "totalNbTasks": 1,
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap
@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "canceledBy": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "error": null,
      "duration": "[duration]",
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap
@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "canceledBy": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "error": null,
      "duration": "[duration]",
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap
@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "canceledBy": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "error": null,
      "duration": "[duration]",
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap
@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "progress": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "stats": {
        "totalNbTasks": 1,
--- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap
+++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap
@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
      "canceledBy": null,
      "details": {
        "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.27.0"
+        "upgradeTo": "v1.26.0"
      },
      "error": null,
      "duration": "[duration]",
--- a/crates/milli/src/documents/sort.rs
+++ b/crates/milli/src/documents/sort.rs
@@ -87,7 +87,7 @@ impl Iterator for SortedDocumentsIterator<'_> {
        };

        // Otherwise don't directly iterate over children, skip them if we know we will go further
-        let mut to_skip = n;
+        let mut to_skip = n - 1;
        while to_skip > 0 {
            if let Err(e) = SortedDocumentsIterator::update_current(
                current_child,
@@ -108,7 +108,7 @@ impl Iterator for SortedDocumentsIterator<'_> {
                continue;
            } else {
                // The current iterator is large enough, so we can forward the call to it.
-                return inner.nth(to_skip);
+                return inner.nth(to_skip + 1);
            }
        }

--- a/crates/milli/src/index.rs
+++ b/crates/milli/src/index.rs
@@ -84,6 +84,7 @@ pub mod main_key {
    pub const SEARCH_CUTOFF: &str = "search_cutoff";
    pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
    pub const FACET_SEARCH: &str = "facet_search";
+    pub const EXECUTE_AFTER_UPDATE: &str = "execute-after-update";
    pub const PREFIX_SEARCH: &str = "prefix_search";
    pub const DOCUMENTS_STATS: &str = "documents_stats";
    pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
@@ -1765,6 +1766,22 @@ impl Index {
        self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
    }

+    pub fn execute_after_update<'t>(&self, txn: &'t RoTxn<'_>) -> heed::Result<Option<&'t str>> {
+        self.main.remap_types::<Str, Str>().get(txn, main_key::EXECUTE_AFTER_UPDATE)
+    }
+
+    pub(crate) fn put_execute_after_update(
+        &self,
+        txn: &mut RwTxn<'_>,
+        val: &str,
+    ) -> heed::Result<()> {
+        self.main.remap_types::<Str, Str>().put(txn, main_key::EXECUTE_AFTER_UPDATE, &val)
+    }
+
+    pub(crate) fn delete_execute_after_update(&self, txn: &mut RwTxn<'_>) -> heed::Result<bool> {
+        self.main.remap_key_type::<Str>().delete(txn, main_key::EXECUTE_AFTER_UPDATE)
+    }
+
    pub fn localized_attributes_rules(
        &self,
        rtxn: &RoTxn<'_>,
--- a/crates/milli/src/update/new/document.rs
+++ b/crates/milli/src/update/new/document.rs
@@ -470,6 +470,71 @@ impl<'doc> Versions<'doc> {
        Ok(Some(Self::single(data)))
    }

+    pub fn multiple_with_edits(
+        doc: Option<rhai::Map>,
+        mut versions: impl Iterator<Item = Result<RawMap<'doc, FxBuildHasher>>>,
+        engine: &rhai::Engine,
+        edit_function: &rhai::AST,
+        doc_alloc: &'doc bumpalo::Bump,
+    ) -> Result<Option<Option<Self>>> {
+        let Some(data) = versions.next() else { return Ok(None) };
+
+        let mut doc = doc.unwrap_or_default();
+        let mut data = data?;
+        for version in versions {
+            let version = version?;
+            for (field, value) in version {
+                data.insert(field, value);
+            }
+
+            let mut scope = rhai::Scope::new();
+            data.iter().for_each(|(k, v)| {
+                doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
+            });
+            scope.push("doc", doc.clone());
+
+            let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
+            data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
+            match scope.get_value::<rhai::Map>("doc") {
+                Some(map) => {
+                    for (key, value) in map {
+                        let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
+                        serde_json::to_writer(&mut vec, &value).unwrap();
+                        let key = doc_alloc.alloc_str(key.as_str());
+                        let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
+                        data.insert(key, raw_value);
+                    }
+                }
+                // In case the deletes the document and it's not the last change
+                // we simply set the document to an empty one and await the next change.
+                None => (),
+            }
+        }
+
+        // We must also run the code after the last change
+        let mut scope = rhai::Scope::new();
+        data.iter().for_each(|(k, v)| {
+            doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
+        });
+        scope.push("doc", doc);
+
+        let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
+        data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
+        match scope.get_value::<rhai::Map>("doc") {
+            Some(map) => {
+                for (key, value) in map {
+                    let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
+                    serde_json::to_writer(&mut vec, &value).unwrap();
+                    let key = doc_alloc.alloc_str(key.as_str());
+                    let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
+                    data.insert(key, raw_value);
+                }
+                Ok(Some(Some(Self::single(data))))
+            }
+            None => Ok(Some(None)),
+        }
+    }
+
    pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
        Self { data: version }
    }
--- a/crates/milli/src/update/new/indexer/document_operation.rs
+++ b/crates/milli/src/update/new/indexer/document_operation.rs
@@ -18,6 +18,7 @@ use crate::documents::PrimaryKey;
 use crate::progress::{AtomicPayloadStep, Progress};
 use crate::update::new::document::{DocumentContext, Versions};
 use crate::update::new::indexer::enterprise_edition::sharding::Shards;
+use crate::update::new::indexer::update_by_function::obkv_to_rhaimap;
 use crate::update::new::steps::IndexingStep;
 use crate::update::new::thread_local::MostlySend;
 use crate::update::new::{DocumentIdentifiers, Insertion, Update};
@@ -162,7 +163,16 @@ impl<'pl> DocumentOperation<'pl> {
            .sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));

        let docids_version_offsets = docids_version_offsets.into_bump_slice();
-        Ok((DocumentOperationChanges { docids_version_offsets }, operations_stats, primary_key))
+        let engine = rhai::Engine::new();
+        // Make sure to correctly setup the engine and remove all settings
+        let ast = index.execute_after_update(rtxn)?.map(|f| engine.compile(f).unwrap());
+        let fidmap = index.fields_ids_map(rtxn)?;
+
+        Ok((
+            DocumentOperationChanges { docids_version_offsets, engine, ast, fidmap },
+            operations_stats,
+            primary_key,
+        ))
    }
 }

@@ -435,7 +445,15 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
        'pl: 'doc,
    {
        let (external_doc, payload_operations) = item;
-        payload_operations.merge(external_doc, &context.doc_alloc)
+        payload_operations.merge(
+            &context.rtxn,
+            context.index,
+            &self.fidmap,
+            &self.engine,
+            self.ast.as_ref(),
+            external_doc,
+            &context.doc_alloc,
+        )
    }

    fn len(&self) -> usize {
@@ -444,6 +462,9 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
 }

 pub struct DocumentOperationChanges<'pl> {
+    engine: rhai::Engine,
+    ast: Option<rhai::AST>,
+    fidmap: FieldsIdsMap,
    docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
 }

@@ -506,10 +527,14 @@ impl<'pl> PayloadOperations<'pl> {
    }

    /// Returns only the most recent version of a document based on the updates from the payloads.
-    ///
-    /// This function is only meant to be used when doing a replacement and not an update.
+    #[allow(clippy::too_many_arguments)]
    fn merge<'doc>(
        &self,
+        rtxn: &heed::RoTxn,
+        index: &Index,
+        fidmap: &FieldsIdsMap,
+        engine: &rhai::Engine,
+        ast: Option<&rhai::AST>,
        external_doc: &'doc str,
        doc_alloc: &'doc Bump,
    ) -> Result<Option<DocumentChange<'doc>>>
@@ -573,9 +598,33 @@ impl<'pl> PayloadOperations<'pl> {
                    Ok(document)
                });

-                let Some(versions) = Versions::multiple(versions)? else { return Ok(None) };
+                let versions = match ast {
+                    Some(ast) => {
+                        let doc = index
+                            .documents
+                            .get(rtxn, &self.docid)?
+                            .map(|obkv| obkv_to_rhaimap(obkv, fidmap))
+                            .transpose()?;
+                        match Versions::multiple_with_edits(doc, versions, engine, ast, doc_alloc)?
+                        {
+                            Some(Some(versions)) => Some(versions),
+                            Some(None) if self.is_new => return Ok(None),
+                            Some(None) => {
+                                return Ok(Some(DocumentChange::Deletion(
+                                    DocumentIdentifiers::create(self.docid, external_doc),
+                                )));
+                            }
+                            None => None,
+                        }
+                    }
+                    None => Versions::multiple(versions)?,
+                };

-                if self.is_new {
+                let Some(versions) = versions else {
+                    return Ok(None);
+                };
+
+                if self.is_new || ast.is_some() {
                    Ok(Some(DocumentChange::Insertion(Insertion::create(
                        self.docid,
                        external_doc,
--- a/crates/milli/src/update/new/indexer/update_by_function.rs
+++ b/crates/milli/src/update/new/indexer/update_by_function.rs
@@ -187,7 +187,7 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> {
    }
 }

-fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
+pub fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
    let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
    let map: Result<rhai::Map> = all_keys
        .iter()
--- a/crates/milli/src/update/new/words_prefix_docids.rs
+++ b/crates/milli/src/update/new/words_prefix_docids.rs
@@ -2,6 +2,7 @@ use std::cell::RefCell;
 use std::collections::BTreeSet;
 use std::io::{BufReader, BufWriter, Read, Seek, Write};
 use std::iter;
+use std::num::NonZeroUsize;

 use hashbrown::HashMap;
 use heed::types::{Bytes, DecodeIgnore, Str};
@@ -345,7 +346,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                            indexes.push(PrefixIntegerEntry {
                                prefix,
                                pos,
-                                serialized_length: Some(buffer.len()),
+                                serialized_length: NonZeroUsize::new(buffer.len()),
                            });
                            file.write_all(&buffer)?;
                        }
@@ -371,7 +372,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                key_buffer.extend_from_slice(&pos.to_be_bytes());
                match serialized_length {
                    Some(serialized_length) => {
-                        buffer.resize(serialized_length, 0);
+                        buffer.resize(serialized_length.get(), 0);
                        file.read_exact(&mut buffer)?;
                        self.prefix_database.remap_data_type::<Bytes>().put(
                            wtxn,
@@ -426,7 +427,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                    index.push(PrefixIntegerEntry {
                        prefix,
                        pos,
-                        serialized_length: Some(buffer.len()),
+                        serialized_length: NonZeroUsize::new(buffer.len()),
                    });
                    file.write_all(buffer)?;
                }
@@ -452,7 +453,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                key_buffer.extend_from_slice(&pos.to_be_bytes());
                match serialized_length {
                    Some(serialized_length) => {
-                        buffer.resize(serialized_length, 0);
+                        buffer.resize(serialized_length.get(), 0);
                        file.read_exact(&mut buffer)?;
                        self.prefix_database.remap_data_type::<Bytes>().put(
                            wtxn,
@@ -475,7 +476,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
 struct PrefixIntegerEntry<'a> {
    prefix: &'a str,
    pos: u16,
-    serialized_length: Option<usize>,
+    serialized_length: Option<NonZeroUsize>,
 }

 /// TODO doc
--- a/crates/milli/src/update/settings.rs
+++ b/crates/milli/src/update/settings.rs
@@ -202,6 +202,7 @@ pub struct Settings<'a, 't, 'i> {
    facet_search: Setting<bool>,
    chat: Setting<ChatSettings>,
    vector_store: Setting<VectorStoreBackend>,
+    execute_after_update: Setting<String>,
 }

 impl<'a, 't, 'i> Settings<'a, 't, 'i> {
@@ -242,6 +243,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
            facet_search: Setting::NotSet,
            chat: Setting::NotSet,
            vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
            indexer_config,
        }
    }
@@ -488,6 +490,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
        self.vector_store = Setting::Reset;
    }

+    pub fn set_execute_after_update(&mut self, value: String) {
+        self.execute_after_update = Setting::Set(value);
+    }
+
+    pub fn reset_execute_after_update(&mut self) {
+        self.execute_after_update = Setting::Reset;
+    }
+
    #[tracing::instrument(
        level = "trace"
        skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
@@ -1059,6 +1069,18 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
        Ok(changed)
    }

+    fn update_execute_after_update(&mut self) -> Result<()> {
+        match self.execute_after_update.as_ref() {
+            Setting::Set(new) => {
+                self.index.put_execute_after_update(self.wtxn, &new).map_err(Into::into)
+            }
+            Setting::Reset => {
+                self.index.delete_execute_after_update(self.wtxn).map(drop).map_err(Into::into)
+            }
+            Setting::NotSet => Ok(()),
+        }
+    }
+
    fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
        match std::mem::take(&mut self.embedder_settings) {
            Setting::Set(configs) => self.update_embedding_configs_set(configs),
@@ -1469,6 +1491,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
        self.update_proximity_precision()?;
        self.update_prefix_search()?;
        self.update_facet_search()?;
+        self.update_execute_after_update()?;
        self.update_localized_attributes_rules()?;
        self.update_disabled_typos_terms()?;
        self.update_chat_config()?;
@@ -1618,6 +1641,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
            disable_on_numbers: Setting::NotSet,
            chat: Setting::NotSet,
            vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
            wtxn: _,
            index: _,
            indexer_config: _,
--- a/crates/milli/src/update/test_settings.rs
+++ b/crates/milli/src/update/test_settings.rs
@@ -899,6 +899,7 @@ fn test_correct_settings_init() {
                disable_on_numbers,
                chat,
                vector_store,
+                execute_after_update,
            } = settings;
            assert!(matches!(searchable_fields, Setting::NotSet));
            assert!(matches!(displayed_fields, Setting::NotSet));
@@ -929,6 +930,7 @@ fn test_correct_settings_init() {
            assert!(matches!(disable_on_numbers, Setting::NotSet));
            assert!(matches!(chat, Setting::NotSet));
            assert!(matches!(vector_store, Setting::NotSet));
+            assert!(matches!(execute_after_update, Setting::NotSet));
        })
        .unwrap();
 }
--- a/crates/milli/src/update/upgrade/mod.rs
+++ b/crates/milli/src/update/upgrade/mod.rs
@@ -44,7 +44,6 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[
    &ToTargetNoOp { target: (1, 24, 0) },
    &ToTargetNoOp { target: (1, 25, 0) },
    &ToTargetNoOp { target: (1, 26, 0) },
-    &ToTargetNoOp { target: (1, 27, 0) },
    // This is the last upgrade function, it will be called when the index is up to date.
    // any other upgrade function should be added before this one.
    &ToCurrentNoOp {},
@@ -82,7 +81,6 @@ const fn start(from: (u32, u32, u32)) -> Option<usize> {
        (1, 24, _) => function_index!(14),
        (1, 25, _) => function_index!(15),
        (1, 26, _) => function_index!(16),
-        (1, 27, _) => function_index!(17),
        // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually
        // considering dumpless upgrade.
        (_major, _minor, _patch) => return None,
Author	SHA1	Message	Date
Clément Renault	5b7bdf3459	Only provide the last batch info	2025-11-20 12:03:11 +01:00
Kerollmops	6e5b42e924	remove-me: Introduce the MEILI_MAX_INDEX_SIZE env variable to limit the size of an index	2025-11-20 11:47:59 +01:00
Kerollmops	03ced42ad3	Fix some errors	2025-11-20 11:47:59 +01:00
Kerollmops	4706440610	Use a NonZeroUsize to store the prefix buffer length	2025-11-20 11:47:58 +01:00
Clément Renault	6700df1319	Fix the retrieval of the doc in scope	2025-11-20 11:47:58 +01:00
Kerollmops	a068931bd9	Allow users to delete documents	2025-11-20 11:47:58 +01:00
Clément Renault	ce87e14aa5	Erase a document if the AST is set	2025-11-20 11:47:58 +01:00
Clément Renault	7ff28985a0	Use the script in the edit documents	2025-11-20 11:47:58 +01:00
Clément Renault	a70aba8b68	Introduce a new executeAfterUpdate index setting	2025-11-20 11:47:58 +01:00
Clément Renault	2d8a61b53a	Seems to work great, still need to read function from settings	2025-11-20 11:47:58 +01:00
Kerollmops	bd1fdc10e6	Expose the three last batches timings	2025-11-20 11:47:58 +01:00
Kerollmops	d1d6e219b3	Update utoipa	2025-11-20 11:47:58 +01:00
Kerollmops	b6a4f917f8	Reset metrics values to keep current steps only	2025-11-20 11:47:58 +01:00
Kerollmops	5c2a431d57	Expose the step currently running on the metrics route	2025-11-20 11:47:58 +01:00
Kerollmops	5781c16957	Simplify the auth filters	2025-11-20 11:47:58 +01:00
Kerollmops	246c44aeb7	Expose the metrics for the last finished batch and not the processing one	2025-11-20 11:47:58 +01:00
Kerollmops	b2ed891f84	Expose batch progress traces on the metrics route	2025-11-20 11:47:57 +01:00