Only provide the last batch info

one

Cargo.lock

@@ -2922,6 +2922,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+
 [[package]]
 name = "hyper"
 version = "1.7.0"
@@ -4020,6 +4026,7 @@ dependencies = [
  "futures",
  "futures-util",
  "hex",
+ "humantime",
  "index-scheduler",
  "indexmap",
  "insta",

crates/dump/src/lib.rs

@@ -341,6 +341,7 @@ pub(crate) mod test {
             prefix_search: Setting::NotSet,
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()

crates/dump/src/reader/compat/v5_to_v6.rs

@@ -423,6 +423,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             prefix_search: v6::Setting::NotSet,
             chat: v6::Setting::NotSet,
             vector_store: v6::Setting::NotSet,
+            execute_after_update: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }

crates/index-scheduler/src/index_mapper/index_map.rs

@@ -306,6 +306,18 @@ fn create_or_open_index(
 ) -> Result<Index> {
     let options = EnvOpenOptions::new();
     let mut options = options.read_txn_without_tls();
+
+    let map_size = match std::env::var("MEILI_MAX_INDEX_SIZE") {
+        Ok(max_size) => {
+            let max_size = max_size.parse().unwrap();
+            map_size.min(max_size)
+        }
+        Err(VarError::NotPresent) => map_size,
+        Err(VarError::NotUnicode(e)) => {
+            panic!("Non unicode max index size in `MEILI_MAX_INDEX_SIZE`: {e:?}")
+        }
+    };
+
     options.map_size(clamp_to_page_size(map_size));
 
     // You can find more details about this experimental

crates/meilisearch-types/src/error.rs

@@ -324,6 +324,7 @@ InvalidSettingsDisplayedAttributes             , InvalidRequest       , BAD_REQU
 InvalidSettingsDistinctAttribute               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsProximityPrecision              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFacetSearch                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsexecuteAfterUpdate              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsPrefixSearch                    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFaceting                        , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFilterableAttributes            , InvalidRequest       , BAD_REQUEST ;

crates/meilisearch-types/src/settings.rs

@@ -326,6 +326,12 @@ pub struct Settings<T> {
     #[schema(value_type = Option<VectorStoreBackend>)]
     pub vector_store: Setting<VectorStoreBackend>,
 
+    /// Function to execute after an update
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsexecuteAfterUpdate>)]
+    #[schema(value_type = Option<String>, example = json!("doc.likes += 1"))]
+    pub execute_after_update: Setting<String>,
+
     #[serde(skip)]
     #[deserr(skip)]
     pub _kind: PhantomData<T>,
@@ -395,6 +401,7 @@ impl Settings<Checked> {
             prefix_search: Setting::Reset,
             chat: Setting::Reset,
             vector_store: Setting::Reset,
+            execute_after_update: Setting::Reset,
             _kind: PhantomData,
         }
     }
@@ -423,6 +430,7 @@ impl Settings<Checked> {
             prefix_search,
             chat,
             vector_store,
+            execute_after_update,
             _kind,
         } = self;
 
@@ -449,6 +457,7 @@ impl Settings<Checked> {
             prefix_search,
             vector_store,
             chat,
+            execute_after_update,
             _kind: PhantomData,
         }
     }
@@ -501,6 +510,7 @@ impl Settings<Unchecked> {
             prefix_search: self.prefix_search,
             chat: self.chat,
             vector_store: self.vector_store,
+            execute_after_update: self.execute_after_update,
             _kind: PhantomData,
         }
     }
@@ -582,6 +592,10 @@ impl Settings<Unchecked> {
             prefix_search: other.prefix_search.or(self.prefix_search),
             chat: other.chat.clone().or(self.chat.clone()),
             vector_store: other.vector_store.or(self.vector_store),
+            execute_after_update: other
+                .execute_after_update
+                .clone()
+                .or(self.execute_after_update.clone()),
             _kind: PhantomData,
         }
     }
@@ -622,6 +636,7 @@ pub fn apply_settings_to_builder(
         prefix_search,
         chat,
         vector_store,
+        execute_after_update,
         _kind,
     } = settings;
 
@@ -845,6 +860,14 @@ pub fn apply_settings_to_builder(
         Setting::Reset => builder.reset_vector_store(),
         Setting::NotSet => (),
     }
+
+    match execute_after_update {
+        Setting::Set(execute_after_update) => {
+            builder.set_execute_after_update(execute_after_update.clone())
+        }
+        Setting::Reset => builder.reset_execute_after_update(),
+        Setting::NotSet => (),
+    }
 }
 
 pub enum SecretPolicy {
@@ -944,13 +967,13 @@ pub fn settings(
         .collect();
 
     let vector_store = index.get_vector_store(rtxn)?;
-
     let embedders = Setting::Set(embedders);
     let search_cutoff_ms = index.search_cutoff(rtxn)?;
     let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
     let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
     let facet_search = index.facet_search(rtxn)?;
     let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
+    let execute_after_update = index.execute_after_update(rtxn)?;
 
     let mut settings = Settings {
         displayed_attributes: match displayed_attributes {
@@ -995,6 +1018,10 @@ pub fn settings(
             Some(vector_store) => Setting::Set(vector_store),
             None => Setting::Reset,
         },
+        execute_after_update: match execute_after_update {
+            Some(function) => Setting::Set(function.to_string()),
+            None => Setting::NotSet,
+        },
         _kind: PhantomData,
     };
 
@@ -1225,6 +1252,7 @@ pub(crate) mod test {
             prefix_search: Setting::NotSet,
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
 
@@ -1258,7 +1286,7 @@ pub(crate) mod test {
             prefix_search: Setting::NotSet,
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
-
+            execute_after_update: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
 

crates/meilisearch/Cargo.toml

@@ -117,7 +117,7 @@ secrecy = "0.10.3"
 actix-web-lab = { version = "0.24.1", default-features = false }
 urlencoding = "2.1.3"
 backoff = { version = "0.4.0", features = ["tokio"] }
-
+humantime = { version = "2.3.0", default-features = false }
 
 [dev-dependencies]
 actix-rt = "2.10.0"

crates/meilisearch/src/metrics.rs

@@ -1,7 +1,8 @@
 use lazy_static::lazy_static;
 use prometheus::{
-    opts, register_gauge, register_histogram_vec, register_int_counter_vec, register_int_gauge,
-    register_int_gauge_vec, Gauge, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
+    opts, register_gauge, register_gauge_vec, register_histogram_vec, register_int_counter_vec,
+    register_int_gauge, register_int_gauge_vec, Gauge, GaugeVec, HistogramVec, IntCounterVec,
+    IntGauge, IntGaugeVec,
 };
 
 lazy_static! {
@@ -73,6 +74,20 @@ lazy_static! {
         &["kind", "value"]
     )
     .expect("Can't create a metric");
+    pub static ref MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE: GaugeVec = register_gauge_vec!(
+        opts!("meilisearch_batch_running_progress_trace", "The currently running progress trace"),
+        &["batch_uid", "step_name"]
+    )
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS: IntGaugeVec =
+        register_int_gauge_vec!(
+            opts!(
+                "meilisearch_last_finished_batches_progress_trace_ms",
+                "The last few batches progress trace in milliseconds"
+            ),
+            &["batch_uid", "step_name"]
+        )
+        .expect("Can't create a metric");
     pub static ref MEILISEARCH_LAST_UPDATE: IntGauge =
         register_int_gauge!(opts!("meilisearch_last_update", "Meilisearch Last Update"))
             .expect("Can't create a metric");

crates/meilisearch/src/routes/indexes/settings.rs

@@ -498,6 +498,17 @@ make_setting_routes!(
         camelcase_attr: "facetSearch",
         analytics: FacetSearchAnalytics
     },
+    {
+        route: "/execute-after-update",
+        update_verb: put,
+        value_type: String,
+        err_type: meilisearch_types::deserr::DeserrJsonError<
+            meilisearch_types::error::deserr_codes::InvalidSettingsexecuteAfterUpdate,
+        >,
+        attr: execute_after_update,
+        camelcase_attr: "executeAfterUpdate",
+        analytics: ExecuteAfterUpdateAnalytics
+    },
     {
         route: "/prefix-search",
         update_verb: put,
@@ -619,6 +630,9 @@ pub async fn update_all(
                 new_settings.non_separator_tokens.as_ref().set(),
             ),
             facet_search: FacetSearchAnalytics::new(new_settings.facet_search.as_ref().set()),
+            execute_after_update: ExecuteAfterUpdateAnalytics::new(
+                new_settings.execute_after_update.as_ref().set(),
+            ),
             prefix_search: PrefixSearchAnalytics::new(new_settings.prefix_search.as_ref().set()),
             chat: ChatAnalytics::new(new_settings.chat.as_ref().set()),
             vector_store: VectorStoreAnalytics::new(new_settings.vector_store.as_ref().set()),

crates/meilisearch/src/routes/indexes/settings_analytics.rs

@@ -42,6 +42,7 @@ pub struct SettingsAnalytics {
     pub prefix_search: PrefixSearchAnalytics,
     pub chat: ChatAnalytics,
     pub vector_store: VectorStoreAnalytics,
+    pub execute_after_update: ExecuteAfterUpdateAnalytics,
 }
 
 impl Aggregate for SettingsAnalytics {
@@ -197,6 +198,9 @@ impl Aggregate for SettingsAnalytics {
                 set: new.facet_search.set | self.facet_search.set,
                 value: new.facet_search.value.or(self.facet_search.value),
             },
+            execute_after_update: ExecuteAfterUpdateAnalytics {
+                set: new.execute_after_update.set | self.execute_after_update.set,
+            },
             prefix_search: PrefixSearchAnalytics {
                 set: new.prefix_search.set | self.prefix_search.set,
                 value: new.prefix_search.value.or(self.prefix_search.value),
@@ -669,6 +673,21 @@ impl FacetSearchAnalytics {
     }
 }
 
+#[derive(Serialize, Default)]
+pub struct ExecuteAfterUpdateAnalytics {
+    pub set: bool,
+}
+
+impl ExecuteAfterUpdateAnalytics {
+    pub fn new(distinct: Option<&String>) -> Self {
+        Self { set: distinct.is_some() }
+    }
+
+    pub fn into_settings(self) -> SettingsAnalytics {
+        SettingsAnalytics { execute_after_update: self, ..Default::default() }
+    }
+}
+
 #[derive(Serialize, Default)]
 pub struct PrefixSearchAnalytics {
     pub set: bool,

crates/meilisearch/src/routes/metrics.rs

@@ -4,6 +4,7 @@ use index_scheduler::{IndexScheduler, Query};
 use meilisearch_auth::AuthController;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::actions;
+use meilisearch_types::milli::progress::ProgressStepView;
 use meilisearch_types::tasks::Status;
 use prometheus::{Encoder, TextEncoder};
 use time::OffsetDateTime;
@@ -38,6 +39,12 @@ pub fn configure(config: &mut web::ServiceConfig) {
 # HELP meilisearch_db_size_bytes Meilisearch DB Size In Bytes
 # TYPE meilisearch_db_size_bytes gauge
 meilisearch_db_size_bytes 1130496
+# HELP meilisearch_batch_running_progress_trace The currently running progress trace
+# TYPE meilisearch_batch_running_progress_trace gauge
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="document"} 0.710618582519409
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="extracting word proximity"} 0.2222222222222222
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="indexing"} 0.6666666666666666
+meilisearch_batch_running_progress_trace{batch_uid="0",step_name="processing tasks"} 0
 # HELP meilisearch_http_requests_total Meilisearch HTTP requests total
 # TYPE meilisearch_http_requests_total counter
 meilisearch_http_requests_total{method="GET",path="/metrics",status="400"} 1
@@ -61,6 +68,13 @@ meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="1
 meilisearch_http_response_time_seconds_bucket{method="GET",path="/metrics",le="+Inf"} 0
 meilisearch_http_response_time_seconds_sum{method="GET",path="/metrics"} 0
 meilisearch_http_response_time_seconds_count{method="GET",path="/metrics"} 0
+# HELP meilisearch_last_finished_batches_progress_trace_ms The last few batches progress trace in milliseconds
+# TYPE meilisearch_last_finished_batches_progress_trace_ms gauge
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks"} 19360
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes"} 368
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes > preparing payloads"} 367
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > computing document changes > preparing payloads > payload"} 367
+meilisearch_last_finished_batches_progress_trace_ms{batch_uid="0",step_name="processing tasks > indexing"} 18970
 # HELP meilisearch_index_count Meilisearch Index Count
 # TYPE meilisearch_index_count gauge
 meilisearch_index_count 1
@@ -148,6 +162,46 @@ pub async fn get_metrics(
         }
     }
 
+    // Fetch and expose the current progressing step
+    crate::metrics::MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE.reset();
+    let (batches, _total) = index_scheduler.get_batches_from_authorized_indexes(
+        &Query { statuses: Some(vec![Status::Processing]), ..Query::default() },
+        auth_filters,
+    )?;
+    if let Some(batch) = batches.into_iter().next() {
+        let batch_uid = batch.uid.to_string();
+        if let Some(progress) = batch.progress {
+            for ProgressStepView { current_step, finished, total } in progress.steps {
+                crate::metrics::MEILISEARCH_BATCH_RUNNING_PROGRESS_TRACE
+                    .with_label_values(&[batch_uid.as_str(), current_step.as_ref()])
+                    // We return the completion ratio of the current step
+                    .set(finished as f64 / total as f64);
+            }
+        }
+    }
+
+    crate::metrics::MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS.reset();
+    let (batches, _total) = index_scheduler.get_batches_from_authorized_indexes(
+        // Fetch the finished batches...
+        &Query { statuses: Some(vec![Status::Succeeded, Status::Failed]), ..Query::default() },
+        auth_filters,
+    )?;
+    // ...and get the last batch only.
+    if let Some(batch) = batches.into_iter().next() {
+        let batch_uid = batch.uid.to_string();
+        for (step_name, duration_str) in batch.stats.progress_trace {
+            let Some(duration_str) = duration_str.as_str() else { continue };
+            match humantime::parse_duration(duration_str) {
+                Ok(duration) => {
+                    crate::metrics::MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS
+                        .with_label_values(&[&batch_uid, &step_name])
+                        .set(duration.as_millis() as i64);
+                }
+                Err(e) => tracing::error!("Failed to parse duration: {e}"),
+            }
+        }
+    }
+
     if let Some(last_update) = response.last_update {
         crate::metrics::MEILISEARCH_LAST_UPDATE.set(last_update.unix_timestamp());
     }

crates/milli/src/index.rs

@@ -84,6 +84,7 @@ pub mod main_key {
     pub const SEARCH_CUTOFF: &str = "search_cutoff";
     pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules";
     pub const FACET_SEARCH: &str = "facet_search";
+    pub const EXECUTE_AFTER_UPDATE: &str = "execute-after-update";
     pub const PREFIX_SEARCH: &str = "prefix_search";
     pub const DOCUMENTS_STATS: &str = "documents_stats";
     pub const DISABLED_TYPOS_TERMS: &str = "disabled_typos_terms";
@@ -1765,6 +1766,22 @@ impl Index {
         self.main.remap_key_type::<Str>().delete(txn, main_key::CHAT)
     }
 
+    pub fn execute_after_update<'t>(&self, txn: &'t RoTxn<'_>) -> heed::Result<Option<&'t str>> {
+        self.main.remap_types::<Str, Str>().get(txn, main_key::EXECUTE_AFTER_UPDATE)
+    }
+
+    pub(crate) fn put_execute_after_update(
+        &self,
+        txn: &mut RwTxn<'_>,
+        val: &str,
+    ) -> heed::Result<()> {
+        self.main.remap_types::<Str, Str>().put(txn, main_key::EXECUTE_AFTER_UPDATE, &val)
+    }
+
+    pub(crate) fn delete_execute_after_update(&self, txn: &mut RwTxn<'_>) -> heed::Result<bool> {
+        self.main.remap_key_type::<Str>().delete(txn, main_key::EXECUTE_AFTER_UPDATE)
+    }
+
     pub fn localized_attributes_rules(
         &self,
         rtxn: &RoTxn<'_>,

crates/milli/src/update/new/document.rs

@@ -470,6 +470,71 @@ impl<'doc> Versions<'doc> {
         Ok(Some(Self::single(data)))
     }
 
+    pub fn multiple_with_edits(
+        doc: Option<rhai::Map>,
+        mut versions: impl Iterator<Item = Result<RawMap<'doc, FxBuildHasher>>>,
+        engine: &rhai::Engine,
+        edit_function: &rhai::AST,
+        doc_alloc: &'doc bumpalo::Bump,
+    ) -> Result<Option<Option<Self>>> {
+        let Some(data) = versions.next() else { return Ok(None) };
+
+        let mut doc = doc.unwrap_or_default();
+        let mut data = data?;
+        for version in versions {
+            let version = version?;
+            for (field, value) in version {
+                data.insert(field, value);
+            }
+
+            let mut scope = rhai::Scope::new();
+            data.iter().for_each(|(k, v)| {
+                doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
+            });
+            scope.push("doc", doc.clone());
+
+            let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
+            data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
+            match scope.get_value::<rhai::Map>("doc") {
+                Some(map) => {
+                    for (key, value) in map {
+                        let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
+                        serde_json::to_writer(&mut vec, &value).unwrap();
+                        let key = doc_alloc.alloc_str(key.as_str());
+                        let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
+                        data.insert(key, raw_value);
+                    }
+                }
+                // In case the deletes the document and it's not the last change
+                // we simply set the document to an empty one and await the next change.
+                None => (),
+            }
+        }
+
+        // We must also run the code after the last change
+        let mut scope = rhai::Scope::new();
+        data.iter().for_each(|(k, v)| {
+            doc.insert(k.into(), serde_json::from_str(v.get()).unwrap());
+        });
+        scope.push("doc", doc);
+
+        let _ = engine.eval_ast_with_scope::<rhai::Dynamic>(&mut scope, edit_function).unwrap();
+        data = RawMap::with_hasher_in(FxBuildHasher, doc_alloc);
+        match scope.get_value::<rhai::Map>("doc") {
+            Some(map) => {
+                for (key, value) in map {
+                    let mut vec = bumpalo::collections::Vec::new_in(doc_alloc);
+                    serde_json::to_writer(&mut vec, &value).unwrap();
+                    let key = doc_alloc.alloc_str(key.as_str());
+                    let raw_value = serde_json::from_slice(vec.into_bump_slice()).unwrap();
+                    data.insert(key, raw_value);
+                }
+                Ok(Some(Some(Self::single(data))))
+            }
+            None => Ok(Some(None)),
+        }
+    }
+
     pub fn single(version: RawMap<'doc, FxBuildHasher>) -> Self {
         Self { data: version }
     }

crates/milli/src/update/new/indexer/document_operation.rs

@@ -18,6 +18,7 @@ use crate::documents::PrimaryKey;
 use crate::progress::{AtomicPayloadStep, Progress};
 use crate::update::new::document::{DocumentContext, Versions};
 use crate::update::new::indexer::enterprise_edition::sharding::Shards;
+use crate::update::new::indexer::update_by_function::obkv_to_rhaimap;
 use crate::update::new::steps::IndexingStep;
 use crate::update::new::thread_local::MostlySend;
 use crate::update::new::{DocumentIdentifiers, Insertion, Update};
@@ -162,7 +163,16 @@ impl<'pl> DocumentOperation<'pl> {
             .sort_unstable_by_key(|(_, po)| first_update_pointer(&po.operations).unwrap_or(0));
 
         let docids_version_offsets = docids_version_offsets.into_bump_slice();
-        Ok((DocumentOperationChanges { docids_version_offsets }, operations_stats, primary_key))
+        let engine = rhai::Engine::new();
+        // Make sure to correctly setup the engine and remove all settings
+        let ast = index.execute_after_update(rtxn)?.map(|f| engine.compile(f).unwrap());
+        let fidmap = index.fields_ids_map(rtxn)?;
+
+        Ok((
+            DocumentOperationChanges { docids_version_offsets, engine, ast, fidmap },
+            operations_stats,
+            primary_key,
+        ))
     }
 }
 
@@ -435,7 +445,15 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
         'pl: 'doc,
     {
         let (external_doc, payload_operations) = item;
-        payload_operations.merge(external_doc, &context.doc_alloc)
+        payload_operations.merge(
+            &context.rtxn,
+            context.index,
+            &self.fidmap,
+            &self.engine,
+            self.ast.as_ref(),
+            external_doc,
+            &context.doc_alloc,
+        )
     }
 
     fn len(&self) -> usize {
@@ -444,6 +462,9 @@ impl<'pl> DocumentChanges<'pl> for DocumentOperationChanges<'pl> {
 }
 
 pub struct DocumentOperationChanges<'pl> {
+    engine: rhai::Engine,
+    ast: Option<rhai::AST>,
+    fidmap: FieldsIdsMap,
     docids_version_offsets: &'pl [(&'pl str, PayloadOperations<'pl>)],
 }
 
@@ -506,10 +527,14 @@ impl<'pl> PayloadOperations<'pl> {
     }
 
     /// Returns only the most recent version of a document based on the updates from the payloads.
-    ///
-    /// This function is only meant to be used when doing a replacement and not an update.
+    #[allow(clippy::too_many_arguments)]
     fn merge<'doc>(
         &self,
+        rtxn: &heed::RoTxn,
+        index: &Index,
+        fidmap: &FieldsIdsMap,
+        engine: &rhai::Engine,
+        ast: Option<&rhai::AST>,
         external_doc: &'doc str,
         doc_alloc: &'doc Bump,
     ) -> Result<Option<DocumentChange<'doc>>>
@@ -573,9 +598,33 @@ impl<'pl> PayloadOperations<'pl> {
                     Ok(document)
                 });
 
-                let Some(versions) = Versions::multiple(versions)? else { return Ok(None) };
+                let versions = match ast {
+                    Some(ast) => {
+                        let doc = index
+                            .documents
+                            .get(rtxn, &self.docid)?
+                            .map(|obkv| obkv_to_rhaimap(obkv, fidmap))
+                            .transpose()?;
+                        match Versions::multiple_with_edits(doc, versions, engine, ast, doc_alloc)?
+                        {
+                            Some(Some(versions)) => Some(versions),
+                            Some(None) if self.is_new => return Ok(None),
+                            Some(None) => {
+                                return Ok(Some(DocumentChange::Deletion(
+                                    DocumentIdentifiers::create(self.docid, external_doc),
+                                )));
+                            }
+                            None => None,
+                        }
+                    }
+                    None => Versions::multiple(versions)?,
+                };
 
-                if self.is_new {
+                let Some(versions) = versions else {
+                    return Ok(None);
+                };
+
+                if self.is_new || ast.is_some() {
                     Ok(Some(DocumentChange::Insertion(Insertion::create(
                         self.docid,
                         external_doc,

crates/milli/src/update/new/indexer/update_by_function.rs

@@ -187,7 +187,7 @@ impl<'index> DocumentChanges<'index> for UpdateByFunctionChanges<'index> {
     }
 }
 
-fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
+pub fn obkv_to_rhaimap(obkv: &KvReaderFieldId, fields_ids_map: &FieldsIdsMap) -> Result<rhai::Map> {
     let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
     let map: Result<rhai::Map> = all_keys
         .iter()

crates/milli/src/update/new/words_prefix_docids.rs

@@ -2,6 +2,7 @@ use std::cell::RefCell;
 use std::collections::BTreeSet;
 use std::io::{BufReader, BufWriter, Read, Seek, Write};
 use std::iter;
+use std::num::NonZeroUsize;
 
 use hashbrown::HashMap;
 use heed::types::{Bytes, DecodeIgnore, Str};
@@ -345,7 +346,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                             indexes.push(PrefixIntegerEntry {
                                 prefix,
                                 pos,
-                                serialized_length: Some(buffer.len()),
+                                serialized_length: NonZeroUsize::new(buffer.len()),
                             });
                             file.write_all(&buffer)?;
                         }
@@ -371,7 +372,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                 key_buffer.extend_from_slice(&pos.to_be_bytes());
                 match serialized_length {
                     Some(serialized_length) => {
-                        buffer.resize(serialized_length, 0);
+                        buffer.resize(serialized_length.get(), 0);
                         file.read_exact(&mut buffer)?;
                         self.prefix_database.remap_data_type::<Bytes>().put(
                             wtxn,
@@ -426,7 +427,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                     index.push(PrefixIntegerEntry {
                         prefix,
                         pos,
-                        serialized_length: Some(buffer.len()),
+                        serialized_length: NonZeroUsize::new(buffer.len()),
                     });
                     file.write_all(buffer)?;
                 }
@@ -452,7 +453,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
                 key_buffer.extend_from_slice(&pos.to_be_bytes());
                 match serialized_length {
                     Some(serialized_length) => {
-                        buffer.resize(serialized_length, 0);
+                        buffer.resize(serialized_length.get(), 0);
                         file.read_exact(&mut buffer)?;
                         self.prefix_database.remap_data_type::<Bytes>().put(
                             wtxn,
@@ -475,7 +476,7 @@ impl<'i> WordPrefixIntegerDocids<'i> {
 struct PrefixIntegerEntry<'a> {
     prefix: &'a str,
     pos: u16,
-    serialized_length: Option<usize>,
+    serialized_length: Option<NonZeroUsize>,
 }
 
 /// TODO doc

crates/milli/src/update/settings.rs

@@ -202,6 +202,7 @@ pub struct Settings<'a, 't, 'i> {
     facet_search: Setting<bool>,
     chat: Setting<ChatSettings>,
     vector_store: Setting<VectorStoreBackend>,
+    execute_after_update: Setting<String>,
 }
 
 impl<'a, 't, 'i> Settings<'a, 't, 'i> {
@@ -242,6 +243,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
             facet_search: Setting::NotSet,
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
             indexer_config,
         }
     }
@@ -488,6 +490,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         self.vector_store = Setting::Reset;
     }
 
+    pub fn set_execute_after_update(&mut self, value: String) {
+        self.execute_after_update = Setting::Set(value);
+    }
+
+    pub fn reset_execute_after_update(&mut self) {
+        self.execute_after_update = Setting::Reset;
+    }
+
     #[tracing::instrument(
         level = "trace"
         skip(self, progress_callback, should_abort, settings_diff, embedder_stats),
@@ -1059,6 +1069,18 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         Ok(changed)
     }
 
+    fn update_execute_after_update(&mut self) -> Result<()> {
+        match self.execute_after_update.as_ref() {
+            Setting::Set(new) => {
+                self.index.put_execute_after_update(self.wtxn, &new).map_err(Into::into)
+            }
+            Setting::Reset => {
+                self.index.delete_execute_after_update(self.wtxn).map(drop).map_err(Into::into)
+            }
+            Setting::NotSet => Ok(()),
+        }
+    }
+
     fn update_embedding_configs(&mut self) -> Result<BTreeMap<String, EmbedderAction>> {
         match std::mem::take(&mut self.embedder_settings) {
             Setting::Set(configs) => self.update_embedding_configs_set(configs),
@@ -1469,6 +1491,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         self.update_proximity_precision()?;
         self.update_prefix_search()?;
         self.update_facet_search()?;
+        self.update_execute_after_update()?;
         self.update_localized_attributes_rules()?;
         self.update_disabled_typos_terms()?;
         self.update_chat_config()?;
@@ -1618,6 +1641,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
             disable_on_numbers: Setting::NotSet,
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
+            execute_after_update: Setting::NotSet,
             wtxn: _,
             index: _,
             indexer_config: _,

crates/milli/src/update/test_settings.rs

@@ -899,6 +899,7 @@ fn test_correct_settings_init() {
                 disable_on_numbers,
                 chat,
                 vector_store,
+                execute_after_update,
             } = settings;
             assert!(matches!(searchable_fields, Setting::NotSet));
             assert!(matches!(displayed_fields, Setting::NotSet));
@@ -929,6 +930,7 @@ fn test_correct_settings_init() {
             assert!(matches!(disable_on_numbers, Setting::NotSet));
             assert!(matches!(chat, Setting::NotSet));
             assert!(matches!(vector_store, Setting::NotSet));
+            assert!(matches!(execute_after_update, Setting::NotSet));
         })
         .unwrap();
 }