Merge #4621

4621: Bring back changes from v1.8.0 into main r=curquiza a=curquiza



Co-authored-by: ManyTheFish <many@meilisearch.com>
Co-authored-by: Tamo <tamo@meilisearch.com>
Co-authored-by: meili-bors[bot] <89034592+meili-bors[bot]@users.noreply.github.com>
Co-authored-by: Clément Renault <clement@meilisearch.com>

milli/src/update/index_documents/extract/extract_docid_word_positions.rs

@@ -1,4 +1,4 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::BufReader;
@@ -12,6 +12,7 @@ use serde_json::Value;
 use super::helpers::{create_sorter, keep_latest_obkv, sorter_into_reader, GrenadParameters};
 use crate::error::{InternalError, SerializationError};
 use crate::update::del_add::{del_add_from_two_obkvs, DelAdd, KvReaderDelAdd};
+use crate::update::settings::{InnerIndexSettings, InnerIndexSettingsDiff};
 use crate::{FieldId, Result, MAX_POSITION_PER_ATTRIBUTE, MAX_WORD_LENGTH};
 
 pub type ScriptLanguageDocidsMap = HashMap<(Script, Language), (RoaringBitmap, RoaringBitmap)>;
@@ -25,10 +26,7 @@ pub type ScriptLanguageDocidsMap = HashMap<(Script, Language), (RoaringBitmap, R
 pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    searchable_fields: &Option<HashSet<FieldId>>,
-    stop_words: Option<&fst::Set<Vec<u8>>>,
-    allowed_separators: Option<&[&str]>,
-    dictionary: Option<&[&str]>,
+    settings_diff: &InnerIndexSettingsDiff,
     max_positions_per_attributes: Option<u32>,
 ) -> Result<(grenad::Reader<BufReader<File>>, ScriptLanguageDocidsMap)> {
     puffin::profile_function!();
@@ -36,6 +34,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     let max_positions_per_attributes = max_positions_per_attributes
         .map_or(MAX_POSITION_PER_ATTRIBUTE, |max| max.min(MAX_POSITION_PER_ATTRIBUTE));
     let max_memory = indexer.max_memory_by_thread();
+    let force_reindexing = settings_diff.reindex_searchable();
 
     // initialize destination values.
     let mut documents_ids = RoaringBitmap::new();
@@ -56,8 +55,37 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     let mut value_buffer = Vec::new();
 
     // initialize tokenizer.
-    let mut builder = tokenizer_builder(stop_words, allowed_separators, dictionary, None);
-    let tokenizer = builder.build();
+    let old_stop_words = settings_diff.old.stop_words.as_ref();
+    let old_separators: Option<Vec<_>> = settings_diff
+        .old
+        .allowed_separators
+        .as_ref()
+        .map(|s| s.iter().map(String::as_str).collect());
+    let old_dictionary: Option<Vec<_>> =
+        settings_diff.old.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
+    let mut del_builder = tokenizer_builder(
+        old_stop_words,
+        old_separators.as_deref(),
+        old_dictionary.as_deref(),
+        None,
+    );
+    let del_tokenizer = del_builder.build();
+
+    let new_stop_words = settings_diff.new.stop_words.as_ref();
+    let new_separators: Option<Vec<_>> = settings_diff
+        .new
+        .allowed_separators
+        .as_ref()
+        .map(|s| s.iter().map(String::as_str).collect());
+    let new_dictionary: Option<Vec<_>> =
+        settings_diff.new.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
+    let mut add_builder = tokenizer_builder(
+        new_stop_words,
+        new_separators.as_deref(),
+        new_dictionary.as_deref(),
+        None,
+    );
+    let add_tokenizer = add_builder.build();
 
     // iterate over documents.
     let mut cursor = obkv_documents.into_cursor()?;
@@ -69,7 +97,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
         let obkv = KvReader::<FieldId>::new(value);
 
         // if the searchable fields didn't change, skip the searchable indexing for this document.
-        if !searchable_fields_changed(&KvReader::<FieldId>::new(value), searchable_fields) {
+        if !force_reindexing && !searchable_fields_changed(&obkv, settings_diff) {
             continue;
         }
 
@@ -85,11 +113,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
                 // deletions
                 lang_safe_tokens_from_document(
                     &obkv,
-                    searchable_fields,
-                    &tokenizer,
-                    stop_words,
-                    allowed_separators,
-                    dictionary,
+                    &settings_diff.old,
+                    &del_tokenizer,
                     max_positions_per_attributes,
                     DelAdd::Deletion,
                     &mut del_buffers,
@@ -99,11 +124,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
                 // additions
                 lang_safe_tokens_from_document(
                     &obkv,
-                    searchable_fields,
-                    &tokenizer,
-                    stop_words,
-                    allowed_separators,
-                    dictionary,
+                    &settings_diff.new,
+                    &add_tokenizer,
                     max_positions_per_attributes,
                     DelAdd::Addition,
                     &mut add_buffers,
@@ -118,8 +140,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
         // transforming two KV<FieldId, KV<u16, String>> into one KV<FieldId, KV<DelAdd, KV<u16, String>>>
         value_buffer.clear();
         del_add_from_two_obkvs(
-            KvReader::<FieldId>::new(del_obkv),
-            KvReader::<FieldId>::new(add_obkv),
+            &KvReader::<FieldId>::new(del_obkv),
+            &KvReader::<FieldId>::new(add_obkv),
             &mut value_buffer,
         )?;
 
@@ -160,8 +182,9 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
 /// Check if any searchable fields of a document changed.
 fn searchable_fields_changed(
     obkv: &KvReader<FieldId>,
-    searchable_fields: &Option<HashSet<FieldId>>,
+    settings_diff: &InnerIndexSettingsDiff,
 ) -> bool {
+    let searchable_fields = &settings_diff.new.searchable_fields_ids;
     for (field_id, field_bytes) in obkv.iter() {
         if searchable_fields.as_ref().map_or(true, |sf| sf.contains(&field_id)) {
             let del_add = KvReaderDelAdd::new(field_bytes);
@@ -206,14 +229,10 @@ fn tokenizer_builder<'a>(
 
 /// Extract words mapped with their positions of a document,
 /// ensuring no Language detection mistakes was made.
-#[allow(clippy::too_many_arguments)] // FIXME: consider grouping arguments in a struct
 fn lang_safe_tokens_from_document<'a>(
     obkv: &KvReader<FieldId>,
-    searchable_fields: &Option<HashSet<FieldId>>,
+    settings: &InnerIndexSettings,
     tokenizer: &Tokenizer,
-    stop_words: Option<&fst::Set<Vec<u8>>>,
-    allowed_separators: Option<&[&str]>,
-    dictionary: Option<&[&str]>,
     max_positions_per_attributes: u32,
     del_add: DelAdd,
     buffers: &'a mut Buffers,
@@ -222,7 +241,7 @@ fn lang_safe_tokens_from_document<'a>(
 
     tokens_from_document(
         obkv,
-        searchable_fields,
+        &settings.searchable_fields_ids,
         tokenizer,
         max_positions_per_attributes,
         del_add,
@@ -246,12 +265,15 @@ fn lang_safe_tokens_from_document<'a>(
         // then we don't rerun the extraction.
         if !script_language.is_empty() {
             // build a new temporary tokenizer including the allow list.
-            let mut builder = tokenizer_builder(
-                stop_words,
-                allowed_separators,
-                dictionary,
-                Some(&script_language),
-            );
+            let stop_words = settings.stop_words.as_ref();
+            let separators: Option<Vec<_>> = settings
+                .allowed_separators
+                .as_ref()
+                .map(|s| s.iter().map(String::as_str).collect());
+            let dictionary: Option<Vec<_>> =
+                settings.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
+            let mut builder =
+                tokenizer_builder(stop_words, separators.as_deref(), dictionary.as_deref(), None);
             let tokenizer = builder.build();
 
             script_language_word_count.clear();
@@ -259,7 +281,7 @@ fn lang_safe_tokens_from_document<'a>(
             // rerun the extraction.
             tokens_from_document(
                 obkv,
-                searchable_fields,
+                &settings.searchable_fields_ids,
                 &tokenizer,
                 max_positions_per_attributes,
                 del_add,
@@ -276,7 +298,7 @@ fn lang_safe_tokens_from_document<'a>(
 /// Extract words mapped with their positions of a document.
 fn tokens_from_document<'a>(
     obkv: &KvReader<FieldId>,
-    searchable_fields: &Option<HashSet<FieldId>>,
+    searchable_fields: &Option<Vec<FieldId>>,
     tokenizer: &Tokenizer,
     max_positions_per_attributes: u32,
     del_add: DelAdd,

milli/src/update/index_documents/extract/extract_facet_number_docids.rs

@@ -10,6 +10,7 @@ use crate::heed_codec::facet::{
     FacetGroupKey, FacetGroupKeyCodec, FieldDocIdFacetF64Codec, OrderedF64Codec,
 };
 use crate::update::del_add::{KvReaderDelAdd, KvWriterDelAdd};
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::Result;
 
 /// Extracts the facet number and the documents ids where this facet number appear.
@@ -20,6 +21,7 @@ use crate::Result;
 pub fn extract_facet_number_docids<R: io::Read + io::Seek>(
     fid_docid_facet_number: grenad::Reader<R>,
     indexer: GrenadParameters,
+    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_facet_string_docids.rs

@@ -15,6 +15,7 @@ use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::index_documents::helpers::{
     merge_deladd_btreeset_string, merge_deladd_cbo_roaring_bitmaps,
 };
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 
 /// Extracts the facet string and the documents ids where this facet string appear.
@@ -25,6 +26,7 @@ use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
     docid_fid_facet_string: grenad::Reader<R>,
     indexer: GrenadParameters,
+    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::BTreeMap;
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::{self, BufReader};
@@ -20,6 +20,7 @@ use crate::error::InternalError;
 use crate::facet::value_encoding::f64_into_bytes;
 use crate::update::del_add::{DelAdd, KvWriterDelAdd};
 use crate::update::index_documents::{create_writer, writer_into_reader};
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{CboRoaringBitmapCodec, DocumentId, Error, FieldId, Result, MAX_FACET_VALUE_LENGTH};
 
 /// The length of the elements that are always in the buffer when inserting new values.
@@ -43,7 +44,7 @@ pub struct ExtractedFacetValues {
 pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    faceted_fields: &HashSet<FieldId>,
+    settings_diff: &InnerIndexSettingsDiff,
     geo_fields_ids: Option<(FieldId, FieldId)>,
 ) -> Result<ExtractedFacetValues> {
     puffin::profile_function!();
@@ -82,7 +83,9 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
         let obkv = obkv::KvReader::new(value);
 
         for (field_id, field_bytes) in obkv.iter() {
-            if faceted_fields.contains(&field_id) {
+            let delete_faceted = settings_diff.old.faceted_fields_ids.contains(&field_id);
+            let add_faceted = settings_diff.new.faceted_fields_ids.contains(&field_id);
+            if delete_faceted || add_faceted {
                 numbers_key_buffer.clear();
                 strings_key_buffer.clear();
 
@@ -99,11 +102,12 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
                 strings_key_buffer.extend_from_slice(docid_bytes);
 
                 let del_add_obkv = obkv::KvReader::new(field_bytes);
-                let del_value = match del_add_obkv.get(DelAdd::Deletion) {
+                let del_value = match del_add_obkv.get(DelAdd::Deletion).filter(|_| delete_faceted)
+                {
                     Some(bytes) => Some(from_slice(bytes).map_err(InternalError::SerdeJson)?),
                     None => None,
                 };
-                let add_value = match del_add_obkv.get(DelAdd::Addition) {
+                let add_value = match del_add_obkv.get(DelAdd::Addition).filter(|_| add_faceted) {
                     Some(bytes) => Some(from_slice(bytes).map_err(InternalError::SerdeJson)?),
                     None => None,
                 };

milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs

@@ -10,6 +10,7 @@ use super::helpers::{
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::Result;
 
 const MAX_COUNTED_WORDS: usize = 30;
@@ -23,6 +24,7 @@ const MAX_COUNTED_WORDS: usize = 30;
 pub fn extract_fid_word_count_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
+    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -17,8 +17,9 @@ use crate::error::UserError;
 use crate::prompt::Prompt;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::index_documents::helpers::try_split_at;
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::vector::Embedder;
-use crate::{DocumentId, FieldsIdsMap, InternalError, Result, VectorOrArrayOfVectors};
+use crate::{DocumentId, InternalError, Result, ThreadPoolNoAbort, VectorOrArrayOfVectors};
 
 /// The length of the elements that are always in the buffer when inserting new values.
 const TRUNCATE_SIZE: usize = size_of::<DocumentId>();
@@ -71,12 +72,15 @@ impl VectorStateDelta {
 pub fn extract_vector_points<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    field_id_map: &FieldsIdsMap,
+    settings_diff: &InnerIndexSettingsDiff,
     prompt: &Prompt,
     embedder_name: &str,
 ) -> Result<ExtractedVectorPoints> {
     puffin::profile_function!();
 
+    let old_fields_ids_map = &settings_diff.old.fields_ids_map;
+    let new_fields_ids_map = &settings_diff.new.fields_ids_map;
+
     // (docid, _index) -> KvWriterDelAdd -> Vector
     let mut manual_vectors_writer = create_writer(
         indexer.chunk_compression_type,
@@ -98,8 +102,6 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
         tempfile::tempfile()?,
     );
 
-    let vectors_fid = field_id_map.id("_vectors");
-
     let mut key_buffer = Vec::new();
     let mut cursor = obkv_documents.into_cursor()?;
     while let Some((key, value)) = cursor.move_on_next()? {
@@ -116,15 +118,29 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
         // lazily get it when needed
         let document_id = || -> Value { from_utf8(external_id_bytes).unwrap().into() };
 
-        let vectors_field = vectors_fid
-            .and_then(|vectors_fid| obkv.get(vectors_fid))
-            .map(KvReaderDelAdd::new)
-            .map(|obkv| to_vector_maps(obkv, document_id))
-            .transpose()?;
+        // the vector field id may have changed
+        let old_vectors_fid = old_fields_ids_map.id("_vectors");
+        // filter the old vector fid if the settings has been changed forcing reindexing.
+        let old_vectors_fid = old_vectors_fid.filter(|_| !settings_diff.reindex_vectors());
 
-        let (del_map, add_map) = vectors_field.unzip();
-        let del_map = del_map.flatten();
-        let add_map = add_map.flatten();
+        let new_vectors_fid = new_fields_ids_map.id("_vectors");
+        let vectors_field = {
+            let del = old_vectors_fid
+                .and_then(|vectors_fid| obkv.get(vectors_fid))
+                .map(KvReaderDelAdd::new)
+                .map(|obkv| to_vector_map(obkv, DelAdd::Deletion, &document_id))
+                .transpose()?
+                .flatten();
+            let add = new_vectors_fid
+                .and_then(|vectors_fid| obkv.get(vectors_fid))
+                .map(KvReaderDelAdd::new)
+                .map(|obkv| to_vector_map(obkv, DelAdd::Addition, &document_id))
+                .transpose()?
+                .flatten();
+            (del, add)
+        };
+
+        let (del_map, add_map) = vectors_field;
 
         let del_value = del_map.and_then(|mut map| map.remove(embedder_name));
         let add_value = add_map.and_then(|mut map| map.remove(embedder_name));
@@ -155,7 +171,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                     VectorStateDelta::NowGenerated(prompt.render(
                         obkv,
                         DelAdd::Addition,
-                        field_id_map,
+                        new_fields_ids_map,
                     )?)
                 } else {
                     VectorStateDelta::NowRemoved
@@ -182,10 +198,16 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
 
                 if document_is_kept {
                     // Don't give up if the old prompt was failing
-                    let old_prompt =
-                        prompt.render(obkv, DelAdd::Deletion, field_id_map).unwrap_or_default();
-                    let new_prompt = prompt.render(obkv, DelAdd::Addition, field_id_map)?;
-                    if old_prompt != new_prompt {
+                    let old_prompt = Some(prompt)
+                        // TODO: this filter works because we erase the vec database when a embedding setting changes.
+                        // When vector pipeline will be optimized, this should be removed.
+                        .filter(|_| !settings_diff.reindex_vectors())
+                        .map(|p| {
+                            p.render(obkv, DelAdd::Deletion, old_fields_ids_map).unwrap_or_default()
+                        });
+                    let new_prompt = prompt.render(obkv, DelAdd::Addition, new_fields_ids_map)?;
+                    if old_prompt.as_ref() != Some(&new_prompt) {
+                        let old_prompt = old_prompt.unwrap_or_default();
                         tracing::trace!(
                             "🚀 Changing prompt from\n{old_prompt}\n===to===\n{new_prompt}"
                         );
@@ -207,6 +229,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
             &mut manual_vectors_writer,
             &mut key_buffer,
             delta,
+            settings_diff,
         )?;
     }
 
@@ -220,15 +243,6 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     })
 }
 
-fn to_vector_maps(
-    obkv: KvReaderDelAdd,
-    document_id: impl Fn() -> Value,
-) -> Result<(Option<serde_json::Map<String, Value>>, Option<serde_json::Map<String, Value>>)> {
-    let del = to_vector_map(obkv, DelAdd::Deletion, &document_id)?;
-    let add = to_vector_map(obkv, DelAdd::Addition, &document_id)?;
-    Ok((del, add))
-}
-
 fn to_vector_map(
     obkv: KvReaderDelAdd,
     side: DelAdd,
@@ -256,10 +270,15 @@ fn push_vectors_diff(
     manual_vectors_writer: &mut Writer<BufWriter<File>>,
     key_buffer: &mut Vec<u8>,
     delta: VectorStateDelta,
+    settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<()> {
     puffin::profile_function!();
     let (must_remove, prompt, (mut del_vectors, mut add_vectors)) = delta.into_values();
-    if must_remove {
+    if must_remove
+    // TODO: the below condition works because we erase the vec database when a embedding setting changes.
+    // When vector pipeline will be optimized, this should be removed.
+    && !settings_diff.reindex_vectors()
+    {
         key_buffer.truncate(TRUNCATE_SIZE);
         remove_vectors_writer.insert(&key_buffer, [])?;
     }
@@ -287,12 +306,16 @@ fn push_vectors_diff(
         match eob {
             EitherOrBoth::Both(_, _) => (), // no need to touch anything
             EitherOrBoth::Left(vector) => {
-                // We insert only the Del part of the Obkv to inform
-                // that we only want to remove all those vectors.
-                let mut obkv = KvWriterDelAdd::memory();
-                obkv.insert(DelAdd::Deletion, cast_slice(&vector))?;
-                let bytes = obkv.into_inner()?;
-                manual_vectors_writer.insert(&key_buffer, bytes)?;
+                // TODO: the below condition works because we erase the vec database when a embedding setting changes.
+                // When vector pipeline will be optimized, this should be removed.
+                if !settings_diff.reindex_vectors() {
+                    // We insert only the Del part of the Obkv to inform
+                    // that we only want to remove all those vectors.
+                    let mut obkv = KvWriterDelAdd::memory();
+                    obkv.insert(DelAdd::Deletion, cast_slice(&vector))?;
+                    let bytes = obkv.into_inner()?;
+                    manual_vectors_writer.insert(&key_buffer, bytes)?;
+                }
             }
             EitherOrBoth::Right(vector) => {
                 // We insert only the Add part of the Obkv to inform
@@ -339,7 +362,7 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
     prompt_reader: grenad::Reader<R>,
     indexer: GrenadParameters,
     embedder: Arc<Embedder>,
-    request_threads: &rayon::ThreadPool,
+    request_threads: &ThreadPoolNoAbort,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
     let n_chunks = embedder.chunk_count_hint(); // chunk level parallelism

milli/src/update/index_documents/extract/extract_word_docids.rs

@@ -1,20 +1,23 @@
-use std::collections::{BTreeSet, HashSet};
+use std::collections::BTreeSet;
 use std::fs::File;
 use std::io::{self, BufReader};
 
-use heed::BytesDecode;
+use heed::{BytesDecode, BytesEncode};
 use obkv::KvReaderU16;
+use roaring::RoaringBitmap;
 
 use super::helpers::{
-    create_sorter, create_writer, merge_deladd_cbo_roaring_bitmaps, sorter_into_reader,
-    try_split_array_at, writer_into_reader, GrenadParameters,
+    create_sorter, create_writer, merge_deladd_cbo_roaring_bitmaps, try_split_array_at,
+    writer_into_reader, GrenadParameters,
 };
 use crate::error::SerializationError;
 use crate::heed_codec::StrBEU16Codec;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{is_noop_del_add_obkv, DelAdd, KvReaderDelAdd, KvWriterDelAdd};
+use crate::update::index_documents::helpers::sorter_into_reader;
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::update::MergeFn;
-use crate::{DocumentId, FieldId, Result};
+use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result};
 
 /// Extracts the word and the documents ids where this word appear.
 ///
@@ -27,7 +30,7 @@ use crate::{DocumentId, FieldId, Result};
 pub fn extract_word_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-    exact_attributes: &HashSet<FieldId>,
+    settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<(
     grenad::Reader<BufReader<File>>,
     grenad::Reader<BufReader<File>>,
@@ -43,7 +46,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|x| x / 3),
+        max_memory.map(|m| m / 3),
     );
     let mut key_buffer = Vec::new();
     let mut del_words = BTreeSet::new();
@@ -85,13 +88,19 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         add_words.clear();
     }
 
+    let mut word_fid_docids_writer = create_writer(
+        indexer.chunk_compression_type,
+        indexer.chunk_compression_level,
+        tempfile::tempfile()?,
+    );
+
     let mut word_docids_sorter = create_sorter(
         grenad::SortAlgorithm::Unstable,
         merge_deladd_cbo_roaring_bitmaps,
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|x| x / 3),
+        max_memory.map(|m| m / 3),
     );
 
     let mut exact_word_docids_sorter = create_sorter(
@@ -100,31 +109,45 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|x| x / 3),
-    );
-
-    let mut word_fid_docids_writer = create_writer(
-        indexer.chunk_compression_type,
-        indexer.chunk_compression_level,
-        tempfile::tempfile()?,
+        max_memory.map(|m| m / 3),
     );
 
     let mut iter = word_fid_docids_sorter.into_stream_merger_iter()?;
-    // TODO: replace sorters by writers by accumulating values into a buffer before inserting them.
+    let mut buffer = Vec::new();
+    // NOTE: replacing sorters by bitmap merging is less efficient, so, use sorters.
     while let Some((key, value)) = iter.next()? {
         // only keep the value if their is a change to apply in the DB.
         if !is_noop_del_add_obkv(KvReaderDelAdd::new(value)) {
             word_fid_docids_writer.insert(key, value)?;
         }
 
-        let (word, fid) = StrBEU16Codec::bytes_decode(key)
+        let (w, fid) = StrBEU16Codec::bytes_decode(key)
             .map_err(|_| SerializationError::Decoding { db_name: Some(DOCID_WORD_POSITIONS) })?;
 
-        // every words contained in an attribute set to exact must be pushed in the exact_words list.
-        if exact_attributes.contains(&fid) {
-            exact_word_docids_sorter.insert(word.as_bytes(), value)?;
-        } else {
-            word_docids_sorter.insert(word.as_bytes(), value)?;
+        // merge all deletions
+        let obkv = KvReaderDelAdd::new(value);
+        if let Some(value) = obkv.get(DelAdd::Deletion) {
+            let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid);
+            buffer.clear();
+            let mut obkv = KvWriterDelAdd::new(&mut buffer);
+            obkv.insert(DelAdd::Deletion, value)?;
+            if delete_from_exact {
+                exact_word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
+            } else {
+                word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
+            }
+        }
+        // merge all additions
+        if let Some(value) = obkv.get(DelAdd::Addition) {
+            let add_in_exact = settings_diff.new.exact_attributes.contains(&fid);
+            buffer.clear();
+            let mut obkv = KvWriterDelAdd::new(&mut buffer);
+            obkv.insert(DelAdd::Addition, value)?;
+            if add_in_exact {
+                exact_word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
+            } else {
+                word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
+            }
         }
     }
 
@@ -178,3 +201,45 @@ fn words_into_sorter(
 
     Ok(())
 }
+
+#[tracing::instrument(level = "trace", skip_all, target = "indexing::extract")]
+fn docids_into_writers<W>(
+    word: &str,
+    deletions: &RoaringBitmap,
+    additions: &RoaringBitmap,
+    writer: &mut grenad::Writer<W>,
+) -> Result<()>
+where
+    W: std::io::Write,
+{
+    if deletions == additions {
+        // if the same value is deleted and added, do nothing.
+        return Ok(());
+    }
+
+    // Write each value in the same KvDelAdd before inserting it in the final writer.
+    let mut obkv = KvWriterDelAdd::memory();
+    // deletions:
+    if !deletions.is_empty() && !deletions.is_subset(additions) {
+        obkv.insert(
+            DelAdd::Deletion,
+            CboRoaringBitmapCodec::bytes_encode(deletions).map_err(|_| {
+                SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) }
+            })?,
+        )?;
+    }
+    // additions:
+    if !additions.is_empty() {
+        obkv.insert(
+            DelAdd::Addition,
+            CboRoaringBitmapCodec::bytes_encode(additions).map_err(|_| {
+                SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) }
+            })?,
+        )?;
+    }
+
+    // insert everything in the same writer.
+    writer.insert(word.as_bytes(), obkv.into_inner().unwrap())?;
+
+    Ok(())
+}

milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs

@@ -11,8 +11,9 @@ use super::helpers::{
 };
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
-use crate::proximity::{index_proximity, MAX_DISTANCE};
+use crate::proximity::{index_proximity, ProximityPrecision, MAX_DISTANCE};
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{DocumentId, Result};
 
 /// Extracts the best proximity between pairs of words and the documents ids where this pair appear.
@@ -23,8 +24,21 @@ use crate::{DocumentId, Result};
 pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
+    settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
+    let any_deletion = settings_diff.old.proximity_precision == ProximityPrecision::ByWord;
+    let any_addition = settings_diff.new.proximity_precision == ProximityPrecision::ByWord;
+
+    // early return if the data shouldn't be deleted nor created.
+    if !any_deletion && !any_addition {
+        let writer = create_writer(
+            indexer.chunk_compression_type,
+            indexer.chunk_compression_level,
+            tempfile::tempfile()?,
+        );
+        return writer_into_reader(writer);
+    }
 
     let max_memory = indexer.max_memory_by_thread();
 
@@ -77,6 +91,10 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
 
         let (del, add): (Result<_>, Result<_>) = rayon::join(
             || {
+                if !any_deletion {
+                    return Ok(());
+                }
+
                 // deletions
                 if let Some(deletion) = KvReaderDelAdd::new(value).get(DelAdd::Deletion) {
                     for (position, word) in KvReaderU16::new(deletion).iter() {
@@ -106,6 +124,10 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
                 Ok(())
             },
             || {
+                if !any_addition {
+                    return Ok(());
+                }
+
                 // additions
                 if let Some(addition) = KvReaderDelAdd::new(value).get(DelAdd::Addition) {
                     for (position, word) in KvReaderU16::new(addition).iter() {

milli/src/update/index_documents/extract/extract_word_position_docids.rs

@@ -11,6 +11,7 @@ use super::helpers::{
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
+use crate::update::settings::InnerIndexSettingsDiff;
 use crate::update::MergeFn;
 use crate::{bucketed_position, DocumentId, Result};
 
@@ -22,6 +23,7 @@ use crate::{bucketed_position, DocumentId, Result};
 pub fn extract_word_position_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
+    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/mod.rs

@@ -9,9 +9,9 @@ mod extract_word_docids;
 mod extract_word_pair_proximity_docids;
 mod extract_word_position_docids;
 
-use std::collections::HashSet;
 use std::fs::File;
 use std::io::BufReader;
+use std::sync::Arc;
 
 use crossbeam_channel::Sender;
 use rayon::prelude::*;
@@ -30,9 +30,8 @@ use self::extract_word_pair_proximity_docids::extract_word_pair_proximity_docids
 use self::extract_word_position_docids::extract_word_position_docids;
 use super::helpers::{as_cloneable_grenad, CursorClonableMmap, GrenadParameters};
 use super::{helpers, TypedChunk};
-use crate::proximity::ProximityPrecision;
-use crate::vector::EmbeddingConfigs;
-use crate::{FieldId, FieldsIdsMap, Result};
+use crate::update::settings::InnerIndexSettingsDiff;
+use crate::{FieldId, Result, ThreadPoolNoAbortBuilder};
 
 /// Extract data for each databases from obkv documents in parallel.
 /// Send data in grenad file over provided Sender.
@@ -43,18 +42,10 @@ pub(crate) fn data_from_obkv_documents(
     flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
-    searchable_fields: Option<HashSet<FieldId>>,
-    faceted_fields: HashSet<FieldId>,
     primary_key_id: FieldId,
     geo_fields_ids: Option<(FieldId, FieldId)>,
-    field_id_map: FieldsIdsMap,
-    stop_words: Option<fst::Set<Vec<u8>>>,
-    allowed_separators: Option<&[&str]>,
-    dictionary: Option<&[&str]>,
+    settings_diff: Arc<InnerIndexSettingsDiff>,
     max_positions_per_attributes: Option<u32>,
-    exact_attributes: HashSet<FieldId>,
-    proximity_precision: ProximityPrecision,
-    embedders: EmbeddingConfigs,
 ) -> Result<()> {
     puffin::profile_function!();
 
@@ -67,8 +58,7 @@ pub(crate) fn data_from_obkv_documents(
                         original_documents_chunk,
                         indexer,
                         lmdb_writer_sx.clone(),
-                        field_id_map.clone(),
-                        embedders.clone(),
+                        settings_diff.clone(),
                     )
                 })
                 .collect::<Result<()>>()
@@ -81,13 +71,9 @@ pub(crate) fn data_from_obkv_documents(
                         flattened_obkv_chunks,
                         indexer,
                         lmdb_writer_sx.clone(),
-                        &searchable_fields,
-                        &faceted_fields,
                         primary_key_id,
                         geo_fields_ids,
-                        &stop_words,
-                        &allowed_separators,
-                        &dictionary,
+                        settings_diff.clone(),
                         max_positions_per_attributes,
                     )
                 })
@@ -100,13 +86,12 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             docid_word_positions_chunk.clone(),
                             indexer,
+                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_fid_word_count_docids,
                             TypedChunk::FieldIdWordCountDocids,
                             "field-id-wordcount-docids",
                         );
-
-                        let exact_attributes = exact_attributes.clone();
                         run_extraction_task::<
                             _,
                             _,
@@ -118,10 +103,9 @@ pub(crate) fn data_from_obkv_documents(
                         >(
                             docid_word_positions_chunk.clone(),
                             indexer,
+                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
-                            move |doc_word_pos, indexer| {
-                                extract_word_docids(doc_word_pos, indexer, &exact_attributes)
-                            },
+                            extract_word_docids,
                             |(
                                 word_docids_reader,
                                 exact_word_docids_reader,
@@ -139,6 +123,7 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             docid_word_positions_chunk.clone(),
                             indexer,
+                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_word_position_docids,
                             TypedChunk::WordPositionDocids,
@@ -152,6 +137,7 @@ pub(crate) fn data_from_obkv_documents(
                         >(
                             fid_docid_facet_strings_chunk.clone(),
                             indexer,
+                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_facet_string_docids,
                             TypedChunk::FieldIdFacetStringDocids,
@@ -161,22 +147,22 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             fid_docid_facet_numbers_chunk.clone(),
                             indexer,
+                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_facet_number_docids,
                             TypedChunk::FieldIdFacetNumberDocids,
                             "field-id-facet-number-docids",
                         );
 
-                        if proximity_precision == ProximityPrecision::ByWord {
-                            run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
-                                docid_word_positions_chunk.clone(),
-                                indexer,
-                                lmdb_writer_sx.clone(),
-                                extract_word_pair_proximity_docids,
-                                TypedChunk::WordPairProximityDocids,
-                                "word-pair-proximity-docids",
-                            );
-                        }
+                        run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
+                            docid_word_positions_chunk.clone(),
+                            indexer,
+                            settings_diff.clone(),
+                            lmdb_writer_sx.clone(),
+                            extract_word_pair_proximity_docids,
+                            TypedChunk::WordPairProximityDocids,
+                            "word-pair-proximity-docids",
+                        );
                     }
 
                     Ok(())
@@ -195,12 +181,17 @@ pub(crate) fn data_from_obkv_documents(
 fn run_extraction_task<FE, FS, M>(
     chunk: grenad::Reader<CursorClonableMmap>,
     indexer: GrenadParameters,
+    settings_diff: Arc<InnerIndexSettingsDiff>,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     extract_fn: FE,
     serialize_fn: FS,
     name: &'static str,
 ) where
-    FE: Fn(grenad::Reader<CursorClonableMmap>, GrenadParameters) -> Result<M>
+    FE: Fn(
+            grenad::Reader<CursorClonableMmap>,
+            GrenadParameters,
+            &InnerIndexSettingsDiff,
+        ) -> Result<M>
         + Sync
         + Send
         + 'static,
@@ -213,7 +204,7 @@ fn run_extraction_task<FE, FS, M>(
         let child_span = tracing::trace_span!(target: "indexing::extract::details", parent: &current_span, "extract_multiple_chunks");
         let _entered = child_span.enter();
         puffin::profile_scope!("extract_multiple_chunks", name);
-        match extract_fn(chunk, indexer) {
+        match extract_fn(chunk, indexer, &settings_diff) {
             Ok(chunk) => {
                 let _ = lmdb_writer_sx.send(Ok(serialize_fn(chunk)));
             }
@@ -230,8 +221,7 @@ fn send_original_documents_data(
     original_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
-    field_id_map: FieldsIdsMap,
-    embedders: EmbeddingConfigs,
+    settings_diff: Arc<InnerIndexSettingsDiff>,
 ) -> Result<()> {
     let original_documents_chunk =
         original_documents_chunk.and_then(|c| unsafe { as_cloneable_grenad(&c) })?;
@@ -239,55 +229,58 @@ fn send_original_documents_data(
     let documents_chunk_cloned = original_documents_chunk.clone();
     let lmdb_writer_sx_cloned = lmdb_writer_sx.clone();
 
-    let request_threads = rayon::ThreadPoolBuilder::new()
+    let request_threads = ThreadPoolNoAbortBuilder::new()
         .num_threads(crate::vector::REQUEST_PARALLELISM)
         .thread_name(|index| format!("embedding-request-{index}"))
         .build()?;
 
-    rayon::spawn(move || {
-        for (name, (embedder, prompt)) in embedders {
-            let result = extract_vector_points(
-                documents_chunk_cloned.clone(),
-                indexer,
-                &field_id_map,
-                &prompt,
-                &name,
-            );
-            match result {
-                Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
-                    let embeddings = match extract_embeddings(
-                        prompts,
-                        indexer,
-                        embedder.clone(),
-                        &request_threads,
-                    ) {
-                        Ok(results) => Some(results),
-                        Err(error) => {
-                            let _ = lmdb_writer_sx_cloned.send(Err(error));
-                            None
-                        }
-                    };
+    if settings_diff.reindex_vectors() || !settings_diff.settings_update_only() {
+        let settings_diff = settings_diff.clone();
+        rayon::spawn(move || {
+            for (name, (embedder, prompt)) in settings_diff.new.embedding_configs.clone() {
+                let result = extract_vector_points(
+                    documents_chunk_cloned.clone(),
+                    indexer,
+                    &settings_diff,
+                    &prompt,
+                    &name,
+                );
+                match result {
+                    Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
+                        let embeddings = match extract_embeddings(
+                            prompts,
+                            indexer,
+                            embedder.clone(),
+                            &request_threads,
+                        ) {
+                            Ok(results) => Some(results),
+                            Err(error) => {
+                                let _ = lmdb_writer_sx_cloned.send(Err(error));
+                                None
+                            }
+                        };
 
-                    if !(remove_vectors.is_empty()
-                        && manual_vectors.is_empty()
-                        && embeddings.as_ref().map_or(true, |e| e.is_empty()))
-                    {
-                        let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
-                            remove_vectors,
-                            embeddings,
-                            expected_dimension: embedder.dimensions(),
-                            manual_vectors,
-                            embedder_name: name,
-                        }));
+                        if !(remove_vectors.is_empty()
+                            && manual_vectors.is_empty()
+                            && embeddings.as_ref().map_or(true, |e| e.is_empty()))
+                        {
+                            let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
+                                remove_vectors,
+                                embeddings,
+                                expected_dimension: embedder.dimensions(),
+                                manual_vectors,
+                                embedder_name: name,
+                            }));
+                        }
+                    }
+
+                    Err(error) => {
+                        let _ = lmdb_writer_sx_cloned.send(Err(error));
                     }
                 }
-
-                Err(error) => {
-                    let _ = lmdb_writer_sx_cloned.send(Err(error));
-                }
             }
-        }
-    });
+        });
+    }
 
     // TODO: create a custom internal error
     let _ = lmdb_writer_sx.send(Ok(TypedChunk::Documents(original_documents_chunk)));
@@ -306,13 +299,9 @@ fn send_and_extract_flattened_documents_data(
     flattened_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
-    searchable_fields: &Option<HashSet<FieldId>>,
-    faceted_fields: &HashSet<FieldId>,
     primary_key_id: FieldId,
     geo_fields_ids: Option<(FieldId, FieldId)>,
-    stop_words: &Option<fst::Set<Vec<u8>>>,
-    allowed_separators: &Option<&[&str]>,
-    dictionary: &Option<&[&str]>,
+    settings_diff: Arc<InnerIndexSettingsDiff>,
     max_positions_per_attributes: Option<u32>,
 ) -> Result<(
     grenad::Reader<CursorClonableMmap>,
@@ -341,10 +330,7 @@ fn send_and_extract_flattened_documents_data(
                     extract_docid_word_positions(
                         flattened_documents_chunk.clone(),
                         indexer,
-                        searchable_fields,
-                        stop_words.as_ref(),
-                        *allowed_separators,
-                        *dictionary,
+                        &settings_diff,
                         max_positions_per_attributes,
                     )?;
 
@@ -367,7 +353,7 @@ fn send_and_extract_flattened_documents_data(
                 } = extract_fid_docid_facet_values(
                     flattened_documents_chunk.clone(),
                     indexer,
-                    faceted_fields,
+                    &settings_diff,
                     geo_fields_ids,
                 )?;