add prompt and context

Cargo.toml

@@ -17,8 +17,7 @@ members = [
     "benchmarks",
     "fuzzers",
     "tracing-trace",
-    "xtask",
-    "build-info",
+    "xtask", "build-info",
 ]
 
 [workspace.package]

dump/src/lib.rs

@@ -256,8 +256,8 @@ pub(crate) mod test {
 
     pub fn create_test_settings() -> Settings<Checked> {
         let settings = Settings {
-            displayed_attributes: Setting::Set(vec![S("race"), S("name")]).into(),
-            searchable_attributes: Setting::Set(vec![S("name"), S("race")]).into(),
+            displayed_attributes: Setting::Set(vec![S("race"), S("name")]),
+            searchable_attributes: Setting::Set(vec![S("name"), S("race")]),
             filterable_attributes: Setting::Set(btreeset! { S("race"), S("age") }),
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,

dump/src/reader/compat/v5_to_v6.rs

@@ -315,8 +315,8 @@ impl From<v5::ResponseError> for v6::ResponseError {
 impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
     fn from(settings: v5::Settings<T>) -> Self {
         v6::Settings {
-            displayed_attributes: v6::Setting::from(settings.displayed_attributes).into(),
-            searchable_attributes: v6::Setting::from(settings.searchable_attributes).into(),
+            displayed_attributes: settings.displayed_attributes.into(),
+            searchable_attributes: settings.searchable_attributes.into(),
             filterable_attributes: settings.filterable_attributes.into(),
             sortable_attributes: settings.sortable_attributes.into(),
             ranking_rules: {

filter-parser/src/lib.rs

@@ -568,7 +568,7 @@ pub mod tests {
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequences
+        // but it also works with other sequencies
         insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
     }
 

index-scheduler/Cargo.toml

@@ -37,7 +37,7 @@ time = { version = "0.3.31", features = [
     "macros",
 ] }
 tracing = "0.1.40"
-ureq = "2.9.7"
+ureq = "2.9.1"
 uuid = { version = "1.6.1", features = ["serde", "v4"] }
 
 [dev-dependencies]

index-scheduler/src/batch.rs

@@ -13,7 +13,7 @@ We can combine the two tasks in a single batch:
 1. import documents X and Y
 
 Processing this batch is functionally equivalent to processing the two
-tasks individually, but should be much faster since we are only performing
+tasks individally, but should be much faster since we are only performing
 one indexing operation.
 */
 

index-scheduler/src/lib.rs

@@ -3041,7 +3041,6 @@ mod tests {
             source: Setting::Set(milli::vector::settings::EmbedderSource::Rest),
             api_key: Setting::Set(S("My super secret")),
             url: Setting::Set(S("http://localhost:7777")),
-            dimensions: Setting::Set(4),
             ..Default::default()
         };
         embedders.insert(S("default"), Setting::Set(embedding_settings));

index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap

@@ -7,7 +7,6 @@ expression: task.details
     "default": {
       "source": "rest",
       "apiKey": "MyXXXX...",
-      "dimensions": 4,
       "url": "http://localhost:7777"
     }
   }

index-scheduler/src/snapshots/index_scheduler__tests__settings_update-3.snap

@@ -6,7 +6,7 @@ expression: embedding_config.embedder_options
   "Rest": {
     "api_key": "My super secret",
     "distribution": null,
-    "dimensions": 4,
+    "dimensions": null,
     "url": "http://localhost:7777",
     "query": null,
     "input_field": [

index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap

@@ -7,7 +7,6 @@ expression: task.details
     "default": {
       "source": "rest",
       "apiKey": "MyXXXX...",
-      "dimensions": 4,
       "url": "http://localhost:7777"
     }
   }

index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: NotSet, document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: NotSet, document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: NotSet, document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: NotSet, searchable_attributes: NotSet, filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: NotSet, document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

meilisearch-types/Cargo.toml

@@ -44,7 +44,6 @@ all-tokenizations = ["milli/all-tokenizations"]
 
 # chinese specialized tokenization
 chinese = ["milli/chinese"]
-chinese-pinyin = ["milli/chinese-pinyin"]
 # hebrew specialized tokenization
 hebrew = ["milli/hebrew"]
 # japanese specialized tokenization
@@ -57,5 +56,3 @@ greek = ["milli/greek"]
 khmer = ["milli/khmer"]
 # allow vietnamese specialized tokenization
 vietnamese = ["milli/vietnamese"]
-# force swedish character recomposition
-swedish-recomposition = ["milli/swedish-recomposition"]

meilisearch-types/src/deserr/mod.rs

@@ -26,7 +26,7 @@ pub type DeserrQueryParamError<C = BadRequest> = DeserrError<DeserrQueryParam, C
 
 /// A request deserialization error.
 ///
-/// The first generic parameter is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
+/// The first generic paramater is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
 /// The second generic parameter is the default error code for the deserialization error, in case it is not given.
 pub struct DeserrError<Format, C: Default + ErrorCode> {
     pub msg: String,

meilisearch-types/src/error.rs

@@ -245,6 +245,9 @@ InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSemanticRatio            , InvalidRequest       , BAD_REQUEST ;
 InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
+InvalidRecommendContext               , InvalidRequest       , BAD_REQUEST ;
+InvalidRecommendId                    , InvalidRequest       , BAD_REQUEST ;
+InvalidRecommendPrompt                , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPostTag         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPreTag          , InvalidRequest       , BAD_REQUEST ;
@@ -308,6 +311,8 @@ MissingFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
 MissingIndexUid                       , InvalidRequest       , BAD_REQUEST ;
 MissingMasterKey                      , Auth                 , UNAUTHORIZED ;
 MissingPayload                        , InvalidRequest       , BAD_REQUEST ;
+MissingPrompt                         , InvalidRequest       , BAD_REQUEST ;
+MissingPromptOrId                     , InvalidRequest       , BAD_REQUEST ;
 MissingSearchHybrid                   , InvalidRequest       , BAD_REQUEST ;
 MissingSwapIndexes                    , InvalidRequest       , BAD_REQUEST ;
 MissingTaskFilters                    , InvalidRequest       , BAD_REQUEST ;

meilisearch-types/src/settings.rs

@@ -3,7 +3,7 @@ use std::convert::Infallible;
 use std::fmt;
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
-use std::ops::{ControlFlow, Deref};
+use std::ops::ControlFlow;
 use std::str::FromStr;
 
 use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
@@ -143,13 +143,21 @@ impl MergeWithError<milli::CriterionError> for DeserrJsonError<InvalidSettingsRa
 )]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct Settings<T> {
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[serde(
+        default,
+        serialize_with = "serialize_with_wildcard",
+        skip_serializing_if = "Setting::is_not_set"
+    )]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsDisplayedAttributes>)]
-    pub displayed_attributes: WildcardSetting,
+    pub displayed_attributes: Setting<Vec<String>>,
 
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[serde(
+        default,
+        serialize_with = "serialize_with_wildcard",
+        skip_serializing_if = "Setting::is_not_set"
+    )]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSearchableAttributes>)]
-    pub searchable_attributes: WildcardSetting,
+    pub searchable_attributes: Setting<Vec<String>>,
 
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsFilterableAttributes>)]
@@ -243,8 +251,8 @@ impl<T> Settings<T> {
 impl Settings<Checked> {
     pub fn cleared() -> Settings<Checked> {
         Settings {
-            displayed_attributes: Setting::Reset.into(),
-            searchable_attributes: Setting::Reset.into(),
+            displayed_attributes: Setting::Reset,
+            searchable_attributes: Setting::Reset,
             filterable_attributes: Setting::Reset,
             sortable_attributes: Setting::Reset,
             ranking_rules: Setting::Reset,
@@ -311,7 +319,7 @@ impl Settings<Checked> {
 
 impl Settings<Unchecked> {
     pub fn check(self) -> Settings<Checked> {
-        let displayed_attributes = match self.displayed_attributes.0 {
+        let displayed_attributes = match self.displayed_attributes {
             Setting::Set(fields) => {
                 if fields.iter().any(|f| f == "*") {
                     Setting::Reset
@@ -322,7 +330,7 @@ impl Settings<Unchecked> {
             otherwise => otherwise,
         };
 
-        let searchable_attributes = match self.searchable_attributes.0 {
+        let searchable_attributes = match self.searchable_attributes {
             Setting::Set(fields) => {
                 if fields.iter().any(|f| f == "*") {
                     Setting::Reset
@@ -334,8 +342,8 @@ impl Settings<Unchecked> {
         };
 
         Settings {
-            displayed_attributes: displayed_attributes.into(),
-            searchable_attributes: searchable_attributes.into(),
+            displayed_attributes,
+            searchable_attributes,
             filterable_attributes: self.filterable_attributes,
             sortable_attributes: self.sortable_attributes,
             ranking_rules: self.ranking_rules,
@@ -404,13 +412,13 @@ pub fn apply_settings_to_builder(
         _kind,
     } = settings;
 
-    match searchable_attributes.deref() {
+    match searchable_attributes {
         Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
         Setting::Reset => builder.reset_searchable_fields(),
         Setting::NotSet => (),
     }
 
-    match displayed_attributes.deref() {
+    match displayed_attributes {
         Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
         Setting::Reset => builder.reset_displayed_fields(),
         Setting::NotSet => (),
@@ -682,13 +690,11 @@ pub fn settings(
         displayed_attributes: match displayed_attributes {
             Some(attrs) => Setting::Set(attrs),
             None => Setting::Reset,
-        }
-        .into(),
+        },
         searchable_attributes: match searchable_attributes {
             Some(attrs) => Setting::Set(attrs),
             None => Setting::Reset,
-        }
-        .into(),
+        },
         filterable_attributes: Setting::Set(filterable_attributes),
         sortable_attributes: Setting::Set(sortable_attributes),
         ranking_rules: Setting::Set(criteria.iter().map(|c| c.clone().into()).collect()),
@@ -842,41 +848,6 @@ impl From<ProximityPrecisionView> for ProximityPrecision {
     }
 }
 
-#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)]
-pub struct WildcardSetting(Setting<Vec<String>>);
-
-impl From<Setting<Vec<String>>> for WildcardSetting {
-    fn from(setting: Setting<Vec<String>>) -> Self {
-        Self(setting)
-    }
-}
-
-impl Serialize for WildcardSetting {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        serialize_with_wildcard(&self.0, serializer)
-    }
-}
-
-impl<E: deserr::DeserializeError> Deserr<E> for WildcardSetting {
-    fn deserialize_from_value<V: deserr::IntoValue>(
-        value: deserr::Value<V>,
-        location: ValuePointerRef<'_>,
-    ) -> Result<Self, E> {
-        Ok(Self(Setting::deserialize_from_value(value, location)?))
-    }
-}
-
-impl std::ops::Deref for WildcardSetting {
-    type Target = Setting<Vec<String>>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
 #[cfg(test)]
 pub(crate) mod test {
     use super::*;
@@ -885,8 +856,8 @@ pub(crate) mod test {
     fn test_setting_check() {
         // test no changes
         let settings = Settings {
-            displayed_attributes: Setting::Set(vec![String::from("hello")]).into(),
-            searchable_attributes: Setting::Set(vec![String::from("hello")]).into(),
+            displayed_attributes: Setting::Set(vec![String::from("hello")]),
+            searchable_attributes: Setting::Set(vec![String::from("hello")]),
             filterable_attributes: Setting::NotSet,
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
@@ -912,9 +883,8 @@ pub(crate) mod test {
         // test wildcard
         // test no changes
         let settings = Settings {
-            displayed_attributes: Setting::Set(vec![String::from("*")]).into(),
-            searchable_attributes: Setting::Set(vec![String::from("hello"), String::from("*")])
-                .into(),
+            displayed_attributes: Setting::Set(vec![String::from("*")]),
+            searchable_attributes: Setting::Set(vec![String::from("hello"), String::from("*")]),
             filterable_attributes: Setting::NotSet,
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
@@ -934,7 +904,7 @@ pub(crate) mod test {
         };
 
         let checked = settings.check();
-        assert_eq!(checked.displayed_attributes, Setting::Reset.into());
-        assert_eq!(checked.searchable_attributes, Setting::Reset.into());
+        assert_eq!(checked.displayed_attributes, Setting::Reset);
+        assert_eq!(checked.searchable_attributes, Setting::Reset);
     }
 }

meilisearch/Cargo.toml

@@ -71,13 +71,13 @@ puffin = { version = "0.16.0", features = ["serialization"] }
 rand = "0.8.5"
 rayon = "1.8.0"
 regex = "1.10.2"
-reqwest = { version = "0.12.4", features = [
+reqwest = { version = "0.11.23", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
-rustls = "0.21.12"
+rustls = "0.21.6"
 rustls-pemfile = "1.0.2"
-segment = { version = "0.2.4", optional = true }
+segment = { version = "0.2.3", optional = true }
 serde = { version = "1.0.195", features = ["derive"] }
 serde_json = { version = "1.0.111", features = ["preserve_order"] }
 sha2 = "0.10.8"
@@ -149,14 +149,12 @@ mini-dashboard = [
     "zip",
 ]
 chinese = ["meilisearch-types/chinese"]
-chinese-pinyin = ["meilisearch-types/chinese-pinyin"]
 hebrew = ["meilisearch-types/hebrew"]
 japanese = ["meilisearch-types/japanese"]
 thai = ["meilisearch-types/thai"]
 greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
 vietnamese = ["meilisearch-types/vietnamese"]
-swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
 
 [package.metadata.mini-dashboard]
 assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.13/build.zip"

meilisearch/src/analytics/mock_analytics.rs

@@ -7,6 +7,7 @@ use serde_json::Value;
 
 use super::{find_user_id, Analytics, DocumentDeletionKind, DocumentFetchKind};
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::routes::tasks::TasksFilterQuery;
 use crate::Opt;
 
 pub struct MockAnalytics {
@@ -85,4 +86,6 @@ impl Analytics for MockAnalytics {
     }
     fn get_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
     fn post_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
+    fn get_tasks(&self, _query: &TasksFilterQuery, _request: &HttpRequest) {}
+    fn health_seen(&self, _request: &HttpRequest) {}
 }

meilisearch/src/analytics/mod.rs

@@ -14,6 +14,7 @@ use platform_dirs::AppDirs;
 use serde_json::Value;
 
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::routes::tasks::TasksFilterQuery;
 
 // if the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
@@ -116,4 +117,10 @@ pub trait Analytics: Sync + Send {
         index_creation: bool,
         request: &HttpRequest,
     );
+
+    // this method should be called to aggregate the get tasks requests.
+    fn get_tasks(&self, query: &TasksFilterQuery, request: &HttpRequest);
+
+    // this method should be called to aggregate a add documents request
+    fn health_seen(&self, request: &HttpRequest);
 }

meilisearch/src/analytics/segment_analytics.rs

@@ -33,6 +33,7 @@ use crate::option::{
 };
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::indexes::facet_search::FacetSearchQuery;
+use crate::routes::tasks::TasksFilterQuery;
 use crate::routes::{create_all_stats, Stats};
 use crate::search::{
     FacetSearchResult, MatchingStrategy, SearchQuery, SearchQueryWithIndex, SearchResult,
@@ -80,6 +81,8 @@ pub enum AnalyticsMsg {
     AggregateUpdateDocuments(DocumentsAggregator),
     AggregateGetFetchDocuments(DocumentsFetchAggregator),
     AggregatePostFetchDocuments(DocumentsFetchAggregator),
+    AggregateTasks(TasksAggregator),
+    AggregateHealth(HealthAggregator),
 }
 
 pub struct SegmentAnalytics {
@@ -149,6 +152,8 @@ impl SegmentAnalytics {
             update_documents_aggregator: DocumentsAggregator::default(),
             get_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
             post_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
+            get_tasks_aggregator: TasksAggregator::default(),
+            health_aggregator: HealthAggregator::default(),
         });
         tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone()));
 
@@ -226,6 +231,16 @@ impl super::Analytics for SegmentAnalytics {
         let aggregate = DocumentsFetchAggregator::from_query(documents_query, request);
         let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFetchDocuments(aggregate));
     }
+
+    fn get_tasks(&self, query: &TasksFilterQuery, request: &HttpRequest) {
+        let aggregate = TasksAggregator::from_query(query, request);
+        let _ = self.sender.try_send(AnalyticsMsg::AggregateTasks(aggregate));
+    }
+
+    fn health_seen(&self, request: &HttpRequest) {
+        let aggregate = HealthAggregator::from_query(request);
+        let _ = self.sender.try_send(AnalyticsMsg::AggregateHealth(aggregate));
+    }
 }
 
 /// This structure represent the `infos` field we send in the analytics.
@@ -379,6 +394,8 @@ pub struct Segment {
     update_documents_aggregator: DocumentsAggregator,
     get_fetch_documents_aggregator: DocumentsFetchAggregator,
     post_fetch_documents_aggregator: DocumentsFetchAggregator,
+    get_tasks_aggregator: TasksAggregator,
+    health_aggregator: HealthAggregator,
 }
 
 impl Segment {
@@ -441,6 +458,8 @@ impl Segment {
                         Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateTasks(agreg)) => self.get_tasks_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateHealth(agreg)) => self.health_aggregator.aggregate(agreg),
                         None => (),
                     }
                 }
@@ -494,6 +513,8 @@ impl Segment {
             update_documents_aggregator,
             get_fetch_documents_aggregator,
             post_fetch_documents_aggregator,
+            get_tasks_aggregator,
+            health_aggregator,
         } = self;
 
         if let Some(get_search) =
@@ -541,6 +562,12 @@ impl Segment {
         {
             let _ = self.batcher.push(post_fetch_documents).await;
         }
+        if let Some(get_tasks) = take(get_tasks_aggregator).into_event(user, "Tasks Seen") {
+            let _ = self.batcher.push(get_tasks).await;
+        }
+        if let Some(health) = take(health_aggregator).into_event(user, "Health Seen") {
+            let _ = self.batcher.push(health).await;
+        }
         let _ = self.batcher.flush().await;
     }
 }
@@ -1476,6 +1503,176 @@ impl DocumentsDeletionAggregator {
     }
 }
 
+#[derive(Default, Serialize)]
+pub struct TasksAggregator {
+    #[serde(skip)]
+    timestamp: Option<OffsetDateTime>,
+
+    // context
+    #[serde(rename = "user-agent")]
+    user_agents: HashSet<String>,
+
+    filtered_by_uid: bool,
+    filtered_by_index_uid: bool,
+    filtered_by_type: bool,
+    filtered_by_status: bool,
+    filtered_by_canceled_by: bool,
+    filtered_by_before_enqueued_at: bool,
+    filtered_by_after_enqueued_at: bool,
+    filtered_by_before_started_at: bool,
+    filtered_by_after_started_at: bool,
+    filtered_by_before_finished_at: bool,
+    filtered_by_after_finished_at: bool,
+    total_received: usize,
+}
+
+impl TasksAggregator {
+    pub fn from_query(query: &TasksFilterQuery, request: &HttpRequest) -> Self {
+        let TasksFilterQuery {
+            limit: _,
+            from: _,
+            uids,
+            index_uids,
+            types,
+            statuses,
+            canceled_by,
+            before_enqueued_at,
+            after_enqueued_at,
+            before_started_at,
+            after_started_at,
+            before_finished_at,
+            after_finished_at,
+        } = query;
+
+        Self {
+            timestamp: Some(OffsetDateTime::now_utc()),
+            user_agents: extract_user_agents(request).into_iter().collect(),
+            filtered_by_uid: uids.is_some(),
+            filtered_by_index_uid: index_uids.is_some(),
+            filtered_by_type: types.is_some(),
+            filtered_by_status: statuses.is_some(),
+            filtered_by_canceled_by: canceled_by.is_some(),
+            filtered_by_before_enqueued_at: before_enqueued_at.is_some(),
+            filtered_by_after_enqueued_at: after_enqueued_at.is_some(),
+            filtered_by_before_started_at: before_started_at.is_some(),
+            filtered_by_after_started_at: after_started_at.is_some(),
+            filtered_by_before_finished_at: before_finished_at.is_some(),
+            filtered_by_after_finished_at: after_finished_at.is_some(),
+            total_received: 1,
+        }
+    }
+
+    /// Aggregate one [TasksAggregator] into another.
+    pub fn aggregate(&mut self, other: Self) {
+        let Self {
+            timestamp,
+            user_agents,
+            total_received,
+            filtered_by_uid,
+            filtered_by_index_uid,
+            filtered_by_type,
+            filtered_by_status,
+            filtered_by_canceled_by,
+            filtered_by_before_enqueued_at,
+            filtered_by_after_enqueued_at,
+            filtered_by_before_started_at,
+            filtered_by_after_started_at,
+            filtered_by_before_finished_at,
+            filtered_by_after_finished_at,
+        } = other;
+
+        if self.timestamp.is_none() {
+            self.timestamp = timestamp;
+        }
+
+        // we can't create a union because there is no `into_union` method
+        for user_agent in user_agents {
+            self.user_agents.insert(user_agent);
+        }
+
+        self.filtered_by_uid |= filtered_by_uid;
+        self.filtered_by_index_uid |= filtered_by_index_uid;
+        self.filtered_by_type |= filtered_by_type;
+        self.filtered_by_status |= filtered_by_status;
+        self.filtered_by_canceled_by |= filtered_by_canceled_by;
+        self.filtered_by_before_enqueued_at |= filtered_by_before_enqueued_at;
+        self.filtered_by_after_enqueued_at |= filtered_by_after_enqueued_at;
+        self.filtered_by_before_started_at |= filtered_by_before_started_at;
+        self.filtered_by_after_started_at |= filtered_by_after_started_at;
+        self.filtered_by_before_finished_at |= filtered_by_before_finished_at;
+        self.filtered_by_after_finished_at |= filtered_by_after_finished_at;
+        self.filtered_by_after_finished_at |= filtered_by_after_finished_at;
+
+        self.total_received = self.total_received.saturating_add(total_received);
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        // if we had no timestamp it means we never encountered any events and
+        // thus we don't need to send this event.
+        let timestamp = self.timestamp?;
+
+        Some(Track {
+            timestamp: Some(timestamp),
+            user: user.clone(),
+            event: event_name.to_string(),
+            properties: serde_json::to_value(self).ok()?,
+            ..Default::default()
+        })
+    }
+}
+
+#[derive(Default, Serialize)]
+pub struct HealthAggregator {
+    #[serde(skip)]
+    timestamp: Option<OffsetDateTime>,
+
+    // context
+    #[serde(rename = "user-agent")]
+    user_agents: HashSet<String>,
+
+    #[serde(rename = "requests.total_received")]
+    total_received: usize,
+}
+
+impl HealthAggregator {
+    pub fn from_query(request: &HttpRequest) -> Self {
+        Self {
+            timestamp: Some(OffsetDateTime::now_utc()),
+            user_agents: extract_user_agents(request).into_iter().collect(),
+            total_received: 1,
+        }
+    }
+
+    /// Aggregate one [HealthAggregator] into another.
+    pub fn aggregate(&mut self, other: Self) {
+        let Self { timestamp, user_agents, total_received } = other;
+
+        if self.timestamp.is_none() {
+            self.timestamp = timestamp;
+        }
+
+        // we can't create a union because there is no `into_union` method
+        for user_agent in user_agents {
+            self.user_agents.insert(user_agent);
+        }
+        self.total_received = self.total_received.saturating_add(total_received);
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        // if we had no timestamp it means we never encountered any events and
+        // thus we don't need to send this event.
+        let timestamp = self.timestamp?;
+
+        Some(Track {
+            timestamp: Some(timestamp),
+            user: user.clone(),
+            event: event_name.to_string(),
+            properties: serde_json::to_value(self).ok()?,
+            ..Default::default()
+        })
+    }
+}
+
 #[derive(Default, Serialize)]
 pub struct DocumentsFetchAggregator {
     #[serde(skip)]

meilisearch/src/error.rs

@@ -23,6 +23,8 @@ pub enum MeilisearchHttpError {
     InvalidContentType(String, Vec<String>),
     #[error("Document `{0}` not found.")]
     DocumentNotFound(String),
+    #[error("Document `{0}` not found.")]
+    InvalidDocumentId(String),
     #[error("Sending an empty filter is forbidden.")]
     EmptyFilter,
     #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
@@ -59,6 +61,10 @@ pub enum MeilisearchHttpError {
     Join(#[from] JoinError),
     #[error("Invalid request: missing `hybrid` parameter when both `q` and `vector` are present.")]
     MissingSearchHybrid,
+    #[error("Invalid request: `prompt` parameter is required when `context` is present.")]
+    RecommendMissingPrompt,
+    #[error("Invalid request: one of the `prompt` or `id` parameters is required.")]
+    RecommendMissingPromptOrId,
 }
 
 impl ErrorCode for MeilisearchHttpError {
@@ -70,6 +76,7 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::MissingPayload(_) => Code::MissingPayload,
             MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,
             MeilisearchHttpError::DocumentNotFound(_) => Code::DocumentNotFound,
+            MeilisearchHttpError::InvalidDocumentId(_) => Code::InvalidDocumentId,
             MeilisearchHttpError::EmptyFilter => Code::InvalidDocumentFilter,
             MeilisearchHttpError::InvalidExpression(_, _) => Code::InvalidSearchFilter,
             MeilisearchHttpError::PayloadTooLarge(_) => Code::PayloadTooLarge,
@@ -86,6 +93,8 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
             MeilisearchHttpError::Join(_) => Code::Internal,
             MeilisearchHttpError::MissingSearchHybrid => Code::MissingSearchHybrid,
+            MeilisearchHttpError::RecommendMissingPrompt => Code::MissingPrompt,
+            MeilisearchHttpError::RecommendMissingPromptOrId => Code::MissingPromptOrId,
         }
     }
 }

meilisearch/src/middleware.rs

@@ -59,12 +59,10 @@ where
             let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
-                let request_pattern = req.match_pattern();
-                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
                 let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, metric_path])
+                        .with_label_values(&[&request_method, request_path])
                         .start_timer(),
                 );
             }

meilisearch/src/option.rs

@@ -13,7 +13,6 @@ use byte_unit::{Byte, ByteError};
 use clap::Parser;
 use meilisearch_types::features::InstanceTogglableFeatures;
 use meilisearch_types::milli::update::IndexerConfig;
-use meilisearch_types::milli::ThreadPoolNoAbortBuilder;
 use rustls::server::{
     AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, ServerSessionMemoryCache,
 };
@@ -667,7 +666,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
     type Error = anyhow::Error;
 
     fn try_from(other: &IndexerOpts) -> Result<Self, Self::Error> {
-        let thread_pool = ThreadPoolNoAbortBuilder::new()
+        let thread_pool = rayon::ThreadPoolBuilder::new()
             .thread_name(|index| format!("indexing-thread:{index}"))
             .num_threads(*other.max_indexing_threads)
             .build()?;

meilisearch/src/routes/indexes/mod.rs

@@ -27,6 +27,7 @@ use crate::Opt;
 
 pub mod documents;
 pub mod facet_search;
+pub mod recommend;
 pub mod search;
 pub mod settings;
 
@@ -48,6 +49,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
             .service(web::scope("/documents").configure(documents::configure))
             .service(web::scope("/search").configure(search::configure))
             .service(web::scope("/facet-search").configure(facet_search::configure))
+            .service(web::scope("/recommend").configure(recommend::configure))
             .service(web::scope("/settings").configure(settings::configure)),
     );
 }
@@ -269,8 +271,12 @@ impl From<index_scheduler::IndexStats> for IndexStats {
 pub async fn get_index_stats(
     index_scheduler: GuardedData<ActionPolicy<{ actions::STATS_GET }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
+    analytics.publish("Stats Seen".to_string(), json!({ "per_index_uid": true }), Some(&req));
+
     let stats = IndexStats::from(index_scheduler.index_stats(&index_uid)?);
 
     debug!(returns = ?stats, "Get index stats");

meilisearch/src/routes/indexes/recommend.rs

@@ -0,0 +1,53 @@
+use actix_web::web::{self, Data};
+use actix_web::{HttpRequest, HttpResponse};
+use deserr::actix_web::AwebJson;
+use index_scheduler::IndexScheduler;
+use meilisearch_types::deserr::DeserrJsonError;
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::index_uid::IndexUid;
+use meilisearch_types::keys::actions;
+use tracing::debug;
+
+use super::ActionPolicy;
+use crate::analytics::Analytics;
+use crate::extractors::authentication::GuardedData;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::search::{perform_recommend, RecommendQuery, SearchKind};
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("").route(web::post().to(SeqHandler(recommend))));
+}
+
+pub async fn recommend(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
+    index_uid: web::Path<String>,
+    params: AwebJson<RecommendQuery, DeserrJsonError>,
+    _req: HttpRequest,
+    _analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
+
+    // TODO analytics
+
+    let query = params.into_inner();
+    debug!(parameters = ?query, "Recommend post");
+
+    let index = index_scheduler.index(&index_uid)?;
+
+    let features = index_scheduler.features();
+
+    features.check_vector("Using the recommend API.")?;
+
+    let (embedder_name, embedder) =
+        SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
+
+    let recommendations = tokio::task::spawn_blocking(move || {
+        perform_recommend(&index, query, embedder_name, embedder)
+    })
+    .await?;
+
+    let recommendations = recommendations?;
+
+    debug!(returns = ?recommendations, "Recommend post");
+    Ok(HttpResponse::Ok().json(recommendations))
+}

meilisearch/src/routes/indexes/settings.rs

@@ -137,8 +137,10 @@ macro_rules! make_setting_route {
                 let settings = settings(&index, &rtxn, meilisearch_types::settings::SecretPolicy::HideSecrets)?;
 
                 debug!(returns = ?settings, "Update settings");
+                let mut json = serde_json::json!(&settings);
+                let val = json[$camelcase_attr].take();
 
-                Ok(HttpResponse::Ok().json(settings.$attr))
+                Ok(HttpResponse::Ok().json(val))
             }
 
             pub fn resources() -> Resource {

meilisearch/src/routes/mod.rs

@@ -8,9 +8,11 @@ use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::settings::{Settings, Unchecked};
 use meilisearch_types::tasks::{Kind, Status, Task, TaskId};
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use time::OffsetDateTime;
 use tracing::debug;
 
+use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::search_queue::SearchQueue;
@@ -294,7 +296,10 @@ pub struct Stats {
 async fn get_stats(
     index_scheduler: GuardedData<ActionPolicy<{ actions::STATS_GET }>, Data<IndexScheduler>>,
     auth_controller: GuardedData<ActionPolicy<{ actions::STATS_GET }>, Data<AuthController>>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
+    analytics.publish("Stats Seen".to_string(), json!({ "per_index_uid": false }), Some(&req));
     let filters = index_scheduler.filters();
 
     let stats = create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), filters)?;
@@ -350,7 +355,11 @@ struct VersionResponse {
 
 async fn get_version(
     _index_scheduler: GuardedData<ActionPolicy<{ actions::VERSION }>, Data<IndexScheduler>>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> HttpResponse {
+    analytics.publish("Version Seen".to_string(), json!(null), Some(&req));
+
     let build_info = build_info::BuildInfo::from_build();
 
     HttpResponse::Ok().json(VersionResponse {
@@ -367,11 +376,21 @@ async fn get_version(
     })
 }
 
+#[derive(Serialize)]
+struct KeysResponse {
+    private: Option<String>,
+    public: Option<String>,
+}
+
 pub async fn get_health(
+    req: HttpRequest,
     index_scheduler: Data<IndexScheduler>,
     auth_controller: Data<AuthController>,
     search_queue: Data<SearchQueue>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
+    analytics.health_seen(&req);
+
     search_queue.health().unwrap();
     index_scheduler.health().unwrap();
     auth_controller.health().unwrap();

meilisearch/src/routes/tasks.rs

@@ -270,8 +270,12 @@ pub struct AllTasks {
 async fn get_tasks(
     index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>,
     params: AwebQueryParameter<TasksFilterQuery, DeserrQueryParamError>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let mut params = params.into_inner();
+    analytics.get_tasks(&params, &req);
+
     // We +1 just to know if there is more after this "page" or not.
     params.limit.0 = params.limit.0.saturating_add(1);
     let limit = params.limit.0;
@@ -294,6 +298,8 @@ async fn get_tasks(
 async fn get_task(
     index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>,
     task_uid: web::Path<String>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let task_uid_string = task_uid.into_inner();
 
@@ -304,6 +310,8 @@ async fn get_task(
         }
     };
 
+    analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req));
+
     let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() };
     let filters = index_scheduler.filters();
     let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?;

meilisearch/src/search.rs

@@ -1,4 +1,3 @@
-use core::fmt;
 use std::cmp::min;
 use std::collections::{BTreeMap, BTreeSet, HashSet};
 use std::str::FromStr;
@@ -40,7 +39,7 @@ pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "<em>".to_string();
 pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
 pub const DEFAULT_SEMANTIC_RATIO: fn() -> SemanticRatio = || SemanticRatio(0.5);
 
-#[derive(Clone, Default, PartialEq, Deserr)]
+#[derive(Debug, Clone, Default, PartialEq, Deserr)]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct SearchQuery {
     #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
@@ -89,110 +88,6 @@ pub struct SearchQuery {
     pub attributes_to_search_on: Option<Vec<String>>,
 }
 
-// Since this structure is logged A LOT we're going to reduce the number of things it logs to the bare minimum.
-// - Only what IS used, we know everything else is set to None so there is no need to print it
-// - Re-order the most important field to debug first
-impl fmt::Debug for SearchQuery {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let Self {
-            q,
-            vector,
-            hybrid,
-            offset,
-            limit,
-            page,
-            hits_per_page,
-            attributes_to_retrieve,
-            attributes_to_crop,
-            crop_length,
-            attributes_to_highlight,
-            show_matches_position,
-            show_ranking_score,
-            show_ranking_score_details,
-            filter,
-            sort,
-            facets,
-            highlight_pre_tag,
-            highlight_post_tag,
-            crop_marker,
-            matching_strategy,
-            attributes_to_search_on,
-        } = self;
-
-        let mut debug = f.debug_struct("SearchQuery");
-
-        // First, everything related to the number of documents to retrieve
-        debug.field("limit", &limit).field("offset", &offset);
-        if let Some(page) = page {
-            debug.field("page", &page);
-        }
-        if let Some(hits_per_page) = hits_per_page {
-            debug.field("hits_per_page", &hits_per_page);
-        }
-
-        // Then, everything related to the queries
-        if let Some(q) = q {
-            debug.field("q", &q);
-        }
-        if let Some(v) = vector {
-            if v.len() < 10 {
-                debug.field("vector", &v);
-            } else {
-                debug.field(
-                    "vector",
-                    &format!("[{}, {}, {}, ... {} dimensions]", v[0], v[1], v[2], v.len()),
-                );
-            }
-        }
-        if let Some(hybrid) = hybrid {
-            debug.field("hybrid", &hybrid);
-        }
-        if let Some(attributes_to_search_on) = attributes_to_search_on {
-            debug.field("attributes_to_search_on", &attributes_to_search_on);
-        }
-        if let Some(filter) = filter {
-            debug.field("filter", &filter);
-        }
-        if let Some(sort) = sort {
-            debug.field("sort", &sort);
-        }
-        if let Some(facets) = facets {
-            debug.field("facets", &facets);
-        }
-        debug.field("matching_strategy", &matching_strategy);
-
-        // Then everything related to the formatting
-        debug.field("crop_length", &crop_length);
-        if *show_matches_position {
-            debug.field("show_matches_position", show_matches_position);
-        }
-        if *show_ranking_score {
-            debug.field("show_ranking_score", show_ranking_score);
-        }
-        if *show_ranking_score_details {
-            debug.field("self.show_ranking_score_details", show_ranking_score_details);
-        }
-        debug.field("crop_length", &crop_length);
-        if let Some(facets) = facets {
-            debug.field("facets", &facets);
-        }
-        if let Some(attributes_to_retrieve) = attributes_to_retrieve {
-            debug.field("attributes_to_retrieve", &attributes_to_retrieve);
-        }
-        if let Some(attributes_to_crop) = attributes_to_crop {
-            debug.field("attributes_to_crop", &attributes_to_crop);
-        }
-        if let Some(attributes_to_highlight) = attributes_to_highlight {
-            debug.field("attributes_to_highlight", &attributes_to_highlight);
-        }
-        debug.field("highlight_pre_tag", &highlight_pre_tag);
-        debug.field("highlight_post_tag", &highlight_post_tag);
-        debug.field("crop_marker", &crop_marker);
-
-        debug.finish()
-    }
-}
-
 #[derive(Debug, Clone, Default, PartialEq, Deserr)]
 #[deserr(error = DeserrJsonError<InvalidHybridQuery>, rename_all = camelCase, deny_unknown_fields)]
 pub struct HybridQuery {
@@ -231,7 +126,7 @@ impl SearchKind {
         Ok(Self::Hybrid { embedder_name, embedder, semantic_ratio })
     }
 
-    fn embedder(
+    pub(crate) fn embedder(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
         embedder_name: Option<&str>,
@@ -417,6 +312,32 @@ impl SearchQueryWithIndex {
     }
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct RecommendQuery {
+    #[deserr(default, error = DeserrJsonError<InvalidRecommendId>)]
+    pub id: Option<String>,
+    #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
+    pub offset: usize,
+    #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
+    pub limit: usize,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchFilter>)]
+    pub filter: Option<Value>,
+    #[deserr(default, error = DeserrJsonError<InvalidEmbedder>, default)]
+    pub embedder: Option<String>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
+    pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchShowRankingScore>, default)]
+    pub show_ranking_score: bool,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchShowRankingScoreDetails>, default)]
+    pub show_ranking_score_details: bool,
+
+    #[deserr(default, error = DeserrJsonError<InvalidRecommendPrompt>)]
+    pub prompt: Option<String>,
+    #[deserr(default, error = DeserrJsonError<InvalidRecommendContext>)]
+    pub context: Option<Value>,
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr)]
 #[deserr(rename_all = camelCase)]
 pub enum MatchingStrategy {
@@ -475,7 +396,7 @@ pub struct SearchHit {
     pub ranking_score_details: Option<serde_json::Map<String, serde_json::Value>>,
 }
 
-#[derive(Serialize, Clone, PartialEq)]
+#[derive(Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
@@ -498,44 +419,15 @@ pub struct SearchResult {
     pub used_negative_operator: bool,
 }
 
-impl fmt::Debug for SearchResult {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let SearchResult {
-            hits,
-            query,
-            processing_time_ms,
-            hits_info,
-            facet_distribution,
-            facet_stats,
-            semantic_hit_count,
-            degraded,
-            used_negative_operator,
-        } = self;
-
-        let mut debug = f.debug_struct("SearchResult");
-        // The most important thing when looking at a search result is the time it took to process
-        debug.field("processing_time_ms", &processing_time_ms);
-        debug.field("hits", &format!("[{} hits returned]", hits.len()));
-        debug.field("query", &query);
-        debug.field("hits_info", &hits_info);
-        if *used_negative_operator {
-            debug.field("used_negative_operator", used_negative_operator);
-        }
-        if *degraded {
-            debug.field("degraded", degraded);
-        }
-        if let Some(facet_distribution) = facet_distribution {
-            debug.field("facet_distribution", &facet_distribution);
-        }
-        if let Some(facet_stats) = facet_stats {
-            debug.field("facet_stats", &facet_stats);
-        }
-        if let Some(semantic_hit_count) = semantic_hit_count {
-            debug.field("semantic_hit_count", &semantic_hit_count);
-        }
-
-        debug.finish()
-    }
+#[derive(Serialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct RecommendResult {
+    pub hits: Vec<SearchHit>,
+    pub id: Option<String>,
+    pub prompt: Option<String>,
+    pub processing_time_ms: u128,
+    #[serde(flatten)]
+    pub hits_info: HitsInfo,
 }
 
 #[derive(Serialize, Debug, Clone, PartialEq)]
@@ -941,6 +833,153 @@ pub fn perform_facet_search(
     })
 }
 
+pub fn perform_recommend(
+    index: &Index,
+    query: RecommendQuery,
+    embedder_name: String,
+    embedder: Arc<Embedder>,
+) -> Result<RecommendResult, MeilisearchHttpError> {
+    let before_search = Instant::now();
+    let rtxn = index.read_txn()?;
+
+    let internal_id = query
+        .id
+        .as_deref()
+        .map(|id| -> Result<_, MeilisearchHttpError> {
+            Ok(index
+                .external_documents_ids()
+                .get(&rtxn, id)?
+                .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(id.to_owned()))?)
+        })
+        .transpose()?;
+
+    let mut recommend = match (query.prompt.as_deref(), internal_id, query.context) {
+        (None, Some(internal_id), None) => milli::Recommend::with_docid(
+            internal_id,
+            query.offset,
+            query.limit,
+            index,
+            &rtxn,
+            embedder_name,
+            embedder,
+        ),
+        (Some(prompt), internal_id, context) => milli::Recommend::with_prompt(
+            prompt,
+            internal_id,
+            context,
+            query.offset,
+            query.limit,
+            index,
+            &rtxn,
+            embedder_name,
+            embedder,
+        ),
+        (None, _, Some(_)) => return Err(MeilisearchHttpError::RecommendMissingPrompt.into()),
+        (None, None, None) => return Err(MeilisearchHttpError::RecommendMissingPromptOrId.into()),
+    };
+
+    if let Some(ref filter) = query.filter {
+        if let Some(facets) = parse_filter(filter)? {
+            recommend.filter(facets);
+        }
+    }
+
+    let milli::SearchResult {
+        documents_ids,
+        matching_words: _,
+        candidates,
+        document_scores,
+        degraded: _,
+        used_negative_operator: _,
+    } = recommend.execute()?;
+
+    let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
+
+    let displayed_ids = index
+        .displayed_fields_ids(&rtxn)?
+        .map(|fields| fields.into_iter().collect::<BTreeSet<_>>())
+        .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
+
+    let fids = |attrs: &BTreeSet<String>| {
+        let mut ids = BTreeSet::new();
+        for attr in attrs {
+            if attr == "*" {
+                ids = displayed_ids.clone();
+                break;
+            }
+
+            if let Some(id) = fields_ids_map.id(attr) {
+                ids.insert(id);
+            }
+        }
+        ids
+    };
+
+    // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default),
+    // but these attributes must be also be present
+    // - in the fields_ids_map
+    // - in the displayed attributes
+    let to_retrieve_ids: BTreeSet<_> = query
+        .attributes_to_retrieve
+        .as_ref()
+        .map(fids)
+        .unwrap_or_else(|| displayed_ids.clone())
+        .intersection(&displayed_ids)
+        .cloned()
+        .collect();
+
+    let mut documents = Vec::new();
+    let documents_iter = index.documents(&rtxn, documents_ids)?;
+
+    for ((_id, obkv), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
+        // First generate a document with all the displayed fields
+        let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
+
+        // select the attributes to retrieve
+        let attributes_to_retrieve = to_retrieve_ids
+            .iter()
+            .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
+        let document =
+            permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
+
+        let ranking_score =
+            query.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
+        let ranking_score_details =
+            query.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
+
+        let hit = SearchHit {
+            document,
+            formatted: Default::default(),
+            matches_position: None,
+            ranking_score_details,
+            ranking_score,
+        };
+        documents.push(hit);
+    }
+
+    let max_total_hits = index
+        .pagination_max_total_hits(&rtxn)
+        .map_err(milli::Error::from)?
+        .map(|x| x as usize)
+        .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
+
+    let number_of_hits = min(candidates.len() as usize, max_total_hits);
+    let hits_info = HitsInfo::OffsetLimit {
+        limit: query.limit,
+        offset: query.offset,
+        estimated_total_hits: number_of_hits,
+    };
+
+    let result = RecommendResult {
+        hits: documents,
+        hits_info,
+        id: query.id,
+        prompt: query.prompt,
+        processing_time_ms: before_search.elapsed().as_millis(),
+    };
+    Ok(result)
+}
+
 fn insert_geo_distance(sorts: &[String], document: &mut Document) {
     lazy_static::lazy_static! {
         static ref GEO_REGEX: Regex =

meilisearch/tests/settings/get_settings.rs

@@ -113,8 +113,7 @@ async fn secrets_are_hidden_in_settings() {
                 "default": {
                     "source": "rest",
                     "url": "https://localhost:7777",
-                    "apiKey": "My super secret value you will never guess",
-                    "dimensions": 4,
+                    "apiKey": "My super secret value you will never guess"
                 }
             }
         }))
@@ -185,7 +184,6 @@ async fn secrets_are_hidden_in_settings() {
         "default": {
           "source": "rest",
           "apiKey": "My suXXXXXX...",
-          "dimensions": 4,
           "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
           "url": "https://localhost:7777",
           "query": null,
@@ -213,7 +211,6 @@ async fn secrets_are_hidden_in_settings() {
         "default": {
           "source": "rest",
           "apiKey": "My suXXXXXX...",
-          "dimensions": 4,
           "url": "https://localhost:7777"
         }
       }

meilitool/src/main.rs

@@ -129,7 +129,7 @@ fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> {
         }
     }
 
-    eprintln!("Successfully deleted {count} content files from disk!");
+    eprintln!("Sucessfully deleted {count} content files from disk!");
 
     Ok(())
 }

milli/Cargo.toml

@@ -17,7 +17,7 @@ bincode = "1.3.3"
 bstr = "1.9.0"
 bytemuck = { version = "1.14.0", features = ["extern_crate_alloc"] }
 byteorder = "1.5.0"
-charabia = { version = "0.8.10", default-features = false }
+charabia = { version = "0.8.8", default-features = false }
 concat-arrays = "0.1.2"
 crossbeam-channel = "0.5.11"
 deserr = "0.6.1"
@@ -26,7 +26,7 @@ flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"
 fxhash = "0.2.1"
 geoutils = "0.5.1"
-grenad = { version = "0.4.6", default-features = false, features = [
+grenad = { version = "0.4.5", default-features = false, features = [
     "rayon",
     "tempfile",
 ] }
@@ -85,7 +85,7 @@ liquid = "0.26.4"
 arroy = "0.2.0"
 rand = "0.8.5"
 tracing = "0.1.40"
-ureq = { version = "2.9.7", features = ["json"] }
+ureq = { version = "2.9.6", features = ["json"] }
 url = "2.5.0"
 
 [dev-dependencies]
@@ -115,7 +115,6 @@ lmdb-posix-sem = ["heed/posix-sem"]
 
 # allow chinese specialized tokenization
 chinese = ["charabia/chinese"]
-chinese-pinyin = ["chinese", "charabia/chinese-normalization-pinyin"]
 
 # allow hebrew specialized tokenization
 hebrew = ["charabia/hebrew"]
@@ -136,11 +135,7 @@ greek = ["charabia/greek"]
 # allow khmer specialized tokenization
 khmer = ["charabia/khmer"]
 
-# allow vietnamese specialized tokenization
 vietnamese = ["charabia/vietnamese"]
 
-# force swedish character recomposition
-swedish-recomposition = ["charabia/swedish-recomposition"]
-
 # allow CUDA support, see <https://github.com/meilisearch/meilisearch/issues/4306>
 cuda = ["candle-core/cuda"]

milli/examples/search.rs

@@ -49,7 +49,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             let start = Instant::now();
 
             let mut ctx = SearchContext::new(&index, &txn);
-            let universe = filtered_universe(&ctx, &None)?;
+            let universe = filtered_universe(ctx.index, ctx.txn, &None)?;
 
             let docs = execute_search(
                 &mut ctx,

milli/src/documents/builder.rs

@@ -203,7 +203,7 @@ fn parse_csv_header(header: &str) -> (&str, AllowedType) {
             "string" => (field_name, AllowedType::String),
             "boolean" => (field_name, AllowedType::Boolean),
             "number" => (field_name, AllowedType::Number),
-            // if the pattern isn't recognized, we keep the whole field.
+            // if the pattern isn't reconized, we keep the whole field.
             _otherwise => (header, AllowedType::String),
         },
         None => (header, AllowedType::String),

milli/src/error.rs

@@ -9,7 +9,6 @@ use serde_json::Value;
 use thiserror::Error;
 
 use crate::documents::{self, DocumentsBatchCursorError};
-use crate::thread_pool_no_abort::PanicCatched;
 use crate::{CriterionError, DocumentId, FieldId, Object, SortError};
 
 pub fn is_reserved_keyword(keyword: &str) -> bool {
@@ -40,19 +39,17 @@ pub enum InternalError {
     Fst(#[from] fst::Error),
     #[error(transparent)]
     DocumentsError(#[from] documents::Error),
-    #[error("Invalid compression type have been specified to grenad")]
+    #[error("Invalid compression type have been specified to grenad.")]
     GrenadInvalidCompressionType,
-    #[error("Invalid grenad file with an invalid version format")]
+    #[error("Invalid grenad file with an invalid version format.")]
     GrenadInvalidFormatVersion,
-    #[error("Invalid merge while processing {process}")]
+    #[error("Invalid merge while processing {process}.")]
     IndexingMergingKeys { process: &'static str },
     #[error("{}", HeedError::InvalidDatabaseTyping)]
     InvalidDatabaseTyping,
     #[error(transparent)]
     RayonThreadPool(#[from] ThreadPoolBuildError),
     #[error(transparent)]
-    PanicInThreadPool(#[from] PanicCatched),
-    #[error(transparent)]
     SerdeJson(#[from] serde_json::Error),
     #[error(transparent)]
     Serialization(#[from] SerializationError),
@@ -60,9 +57,9 @@ pub enum InternalError {
     Store(#[from] MdbError),
     #[error(transparent)]
     Utf8(#[from] str::Utf8Error),
-    #[error("An indexation process was explicitly aborted")]
+    #[error("An indexation process was explicitly aborted.")]
     AbortedIndexation,
-    #[error("The matching words list contains at least one invalid member")]
+    #[error("The matching words list contains at least one invalid member.")]
     InvalidMatchingWords,
     #[error(transparent)]
     ArroyError(#[from] arroy::Error),

milli/src/index.rs

@@ -678,23 +678,6 @@ impl Index {
             .get(rtxn, main_key::USER_DEFINED_SEARCHABLE_FIELDS_KEY)
     }
 
-    /// Identical to `user_defined_searchable_fields`, but returns ids instead.
-    pub fn user_defined_searchable_fields_ids(&self, rtxn: &RoTxn) -> Result<Option<Vec<FieldId>>> {
-        match self.user_defined_searchable_fields(rtxn)? {
-            Some(fields) => {
-                let fields_ids_map = self.fields_ids_map(rtxn)?;
-                let mut fields_ids = Vec::new();
-                for name in fields {
-                    if let Some(field_id) = fields_ids_map.id(name) {
-                        fields_ids.push(field_id);
-                    }
-                }
-                Ok(Some(fields_ids))
-            }
-            None => Ok(None),
-        }
-    }
-
     /* filterable fields */
 
     /// Writes the filterable fields names in the database.
@@ -841,11 +824,11 @@ impl Index {
 
     /// Identical to `user_defined_faceted_fields`, but returns ids instead.
     pub fn user_defined_faceted_fields_ids(&self, rtxn: &RoTxn) -> Result<HashSet<FieldId>> {
-        let fields = self.user_defined_faceted_fields(rtxn)?;
+        let fields = self.faceted_fields(rtxn)?;
         let fields_ids_map = self.fields_ids_map(rtxn)?;
 
         let mut fields_ids = HashSet::new();
-        for name in fields {
+        for name in fields.into_iter() {
             if let Some(field_id) = fields_ids_map.id(&name) {
                 fields_ids.insert(field_id);
             }

milli/src/lib.rs

@@ -21,7 +21,6 @@ pub mod prompt;
 pub mod proximity;
 pub mod score_details;
 mod search;
-mod thread_pool_no_abort;
 pub mod update;
 pub mod vector;
 
@@ -43,7 +42,6 @@ pub use search::new::{
     SearchLogger, VisualSearchLogger,
 };
 use serde_json::Value;
-pub use thread_pool_no_abort::{PanicCatched, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder};
 pub use {charabia as tokenizer, heed};
 
 pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError};
@@ -61,6 +59,7 @@ pub use self::heed_codec::{
 };
 pub use self::index::Index;
 pub use self::search::facet::{FacetValueHit, SearchForFacetValues};
+pub use self::search::recommend::Recommend;
 pub use self::search::{
     FacetDistribution, Filter, FormatOptions, MatchBounds, MatcherBuilder, MatchingWords, OrderBy,
     Search, SearchResult, SemanticSearch, TermsMatchingStrategy, DEFAULT_VALUES_PER_FACET,
@@ -130,7 +129,7 @@ impl fmt::Debug for TimeBudget {
 
 impl Default for TimeBudget {
     fn default() -> Self {
-        Self::new(std::time::Duration::from_millis(1500))
+        Self::new(std::time::Duration::from_millis(150))
     }
 }
 

milli/src/prompt/document.rs

@@ -29,7 +29,7 @@ impl ParsedValue {
 }
 
 impl<'a> Document<'a> {
-    pub fn new(
+    pub fn from_deladd_obkv(
         data: obkv::KvReaderU16<'a>,
         side: DelAdd,
         inverted_field_map: &'a FieldsIdsMap,
@@ -48,6 +48,20 @@ impl<'a> Document<'a> {
         Self(out_data)
     }
 
+    pub fn from_doc_obkv(
+        data: obkv::KvReaderU16<'a>,
+        inverted_field_map: &'a FieldsIdsMap,
+    ) -> Self {
+        let mut out_data = BTreeMap::new();
+        for (fid, raw) in data {
+            let Some(name) = inverted_field_map.name(fid) else {
+                continue;
+            };
+            out_data.insert(name, (raw, ParsedValue::empty()));
+        }
+        Self(out_data)
+    }
+
     fn is_empty(&self) -> bool {
         self.0.is_empty()
     }

milli/src/prompt/mod.rs

@@ -2,6 +2,7 @@ mod context;
 mod document;
 pub(crate) mod error;
 mod fields;
+pub mod recommend;
 mod template_checker;
 
 use std::convert::TryFrom;
@@ -9,7 +10,7 @@ use std::convert::TryFrom;
 use error::{NewPromptError, RenderPromptError};
 
 use self::context::Context;
-use self::document::Document;
+pub use self::document::Document;
 use crate::update::del_add::DelAdd;
 use crate::FieldsIdsMap;
 
@@ -95,7 +96,7 @@ impl Prompt {
         side: DelAdd,
         field_id_map: &FieldsIdsMap,
     ) -> Result<String, RenderPromptError> {
-        let document = Document::new(document, side, field_id_map);
+        let document = Document::from_deladd_obkv(document, side, field_id_map);
         let context = Context::new(&document, field_id_map);
 
         self.template.render(&context).map_err(RenderPromptError::missing_context)

milli/src/prompt/recommend.rs

@@ -0,0 +1,112 @@
+use liquid::model::{
+    DisplayCow, KStringCow, ObjectRender, ObjectSource, State, Value as LiquidValue,
+};
+use liquid::{ObjectView, ValueView};
+
+use super::document::Document;
+
+#[derive(Clone, Debug)]
+pub struct Context<'a> {
+    document: Option<&'a Document<'a>>,
+    context: Option<liquid::Object>,
+}
+
+impl<'a> Context<'a> {
+    pub fn new(document: Option<&'a Document<'a>>, context: Option<serde_json::Value>) -> Self {
+        /// FIXME: unwrap
+        let context = context.map(|context| liquid::to_object(&context).unwrap());
+        Self { document, context }
+    }
+}
+
+impl<'a> ObjectView for Context<'a> {
+    fn as_value(&self) -> &dyn ValueView {
+        self
+    }
+
+    fn size(&self) -> i64 {
+        match (self.context.as_ref(), self.document.as_ref()) {
+            (None, None) => 0,
+            (None, Some(_)) => 1,
+            (Some(_), None) => 1,
+            (Some(_), Some(_)) => 2,
+        }
+    }
+
+    fn keys<'k>(&'k self) -> Box<dyn Iterator<Item = KStringCow<'k>> + 'k> {
+        let keys = match (self.context.as_ref(), self.document.as_ref()) {
+            (None, None) => [].as_slice(),
+            (None, Some(_)) => ["doc"].as_slice(),
+            (Some(_), None) => ["context"].as_slice(),
+            (Some(_), Some(_)) => ["context", "doc"].as_slice(),
+        };
+
+        Box::new(keys.iter().map(|s| KStringCow::from_static(s)))
+    }
+
+    fn values<'k>(&'k self) -> Box<dyn Iterator<Item = &'k dyn ValueView> + 'k> {
+        Box::new(
+            self.context
+                .as_ref()
+                .map(|context| context.as_value())
+                .into_iter()
+                .chain(self.document.map(|document| document.as_value()).into_iter()),
+        )
+    }
+
+    fn iter<'k>(&'k self) -> Box<dyn Iterator<Item = (KStringCow<'k>, &'k dyn ValueView)> + 'k> {
+        Box::new(self.keys().zip(self.values()))
+    }
+
+    fn contains_key(&self, index: &str) -> bool {
+        index == "context" || index == "doc"
+    }
+
+    fn get<'s>(&'s self, index: &str) -> Option<&'s dyn ValueView> {
+        match index {
+            "context" => self.context.as_ref().map(|context| context.as_value()),
+            "doc" => self.document.as_ref().map(|doc| doc.as_value()),
+            _ => None,
+        }
+    }
+}
+
+impl<'a> ValueView for Context<'a> {
+    fn as_debug(&self) -> &dyn std::fmt::Debug {
+        self
+    }
+
+    fn render(&self) -> liquid::model::DisplayCow<'_> {
+        DisplayCow::Owned(Box::new(ObjectRender::new(self)))
+    }
+
+    fn source(&self) -> liquid::model::DisplayCow<'_> {
+        DisplayCow::Owned(Box::new(ObjectSource::new(self)))
+    }
+
+    fn type_name(&self) -> &'static str {
+        "object"
+    }
+
+    fn query_state(&self, state: liquid::model::State) -> bool {
+        match state {
+            State::Truthy => true,
+            State::DefaultValue | State::Empty | State::Blank => false,
+        }
+    }
+
+    fn to_kstr(&self) -> liquid::model::KStringCow<'_> {
+        let s = ObjectRender::new(self).to_string();
+        KStringCow::from_string(s)
+    }
+
+    fn to_value(&self) -> LiquidValue {
+        LiquidValue::Object(
+            self.iter().map(|(k, x)| (k.to_string().into(), x.to_value())).collect(),
+        )
+    }
+
+    fn as_object(&self) -> Option<&dyn ObjectView> {
+        Some(self)
+    }
+}

milli/src/search/facet/facet_distribution.rs

@@ -97,7 +97,6 @@ impl<'a> FacetDistribution<'a> {
     ) -> heed::Result<()> {
         match facet_type {
             FacetType::Number => {
-                let mut lexicographic_distribution = BTreeMap::new();
                 let mut key_buffer: Vec<_> = field_id.to_be_bytes().to_vec();
 
                 let distribution_prelength = distribution.len();
@@ -112,17 +111,14 @@ impl<'a> FacetDistribution<'a> {
 
                     for result in iter {
                         let ((_, _, value), ()) = result?;
-                        *lexicographic_distribution.entry(value.to_string()).or_insert(0) += 1;
+                        *distribution.entry(value.to_string()).or_insert(0) += 1;
 
-                        if lexicographic_distribution.len() - distribution_prelength
-                            == self.max_values_per_facet
+                        if distribution.len() - distribution_prelength == self.max_values_per_facet
                         {
                             break;
                         }
                     }
                 }
-
-                distribution.extend(lexicographic_distribution);
             }
             FacetType::String => {
                 let mut normalized_distribution = BTreeMap::new();

milli/src/search/mod.rs

@@ -24,6 +24,7 @@ pub mod facet;
 mod fst_utils;
 pub mod hybrid;
 pub mod new;
+pub mod recommend;
 
 #[derive(Debug, Clone)]
 pub struct SemanticSearch {
@@ -148,7 +149,7 @@ impl<'a> Search<'a> {
     pub fn execute_for_candidates(&self, has_vector_search: bool) -> Result<RoaringBitmap> {
         if has_vector_search {
             let ctx = SearchContext::new(self.index, self.rtxn);
-            filtered_universe(&ctx, &self.filter)
+            filtered_universe(ctx.index, ctx.txn, &self.filter)
         } else {
             Ok(self.execute()?.candidates)
         }
@@ -161,7 +162,7 @@ impl<'a> Search<'a> {
             ctx.searchable_attributes(searchable_attributes)?;
         }
 
-        let universe = filtered_universe(&ctx, &self.filter)?;
+        let universe = filtered_universe(ctx.index, ctx.txn, &self.filter)?;
         let PartialSearchResult {
             located_query_terms,
             candidates,

milli/src/search/new/geo_sort.rs

@@ -42,7 +42,7 @@ fn facet_number_values<'a>(
 }
 
 /// Define the strategy used by the geo sort.
-/// The parameter represents the cache size, and, in the case of the Dynamic strategy,
+/// The paramater represents the cache size, and, in the case of the Dynamic strategy,
 /// the point where we move from using the iterative strategy to the rtree.
 #[derive(Debug, Clone, Copy)]
 pub enum Strategy {

milli/src/search/new/matches/mod.rs

@@ -134,7 +134,7 @@ impl<'t> Matcher<'t, '_> {
             for (token_position, word_position, word) in words_positions {
                 partial = match partial.match_token(word) {
                     // token matches the partial match, but the match is not full,
-                    // we temporarily save the current token then we try to match the next one.
+                    // we temporarly save the current token then we try to match the next one.
                     Some(MatchType::Partial(partial)) => {
                         potential_matches.push((token_position, word_position, partial.char_len()));
                         partial
@@ -507,7 +507,7 @@ mod tests {
     impl<'a> MatcherBuilder<'a> {
         fn new_test(rtxn: &'a heed::RoTxn, index: &'a TempIndex, query: &str) -> Self {
             let mut ctx = SearchContext::new(index, rtxn);
-            let universe = filtered_universe(&ctx, &None).unwrap();
+            let universe = filtered_universe(ctx.index, ctx.txn, &None).unwrap();
             let crate::search::PartialSearchResult { located_query_terms, .. } = execute_search(
                 &mut ctx,
                 Some(query),
@@ -722,7 +722,7 @@ mod tests {
             @"…void void void void void split the world void void"
         );
 
-        // Text containing matches with different density.
+        // Text containing matches with diferent density.
         let text = "split void the void void world void void void void void void void void void void split the world void void";
         let mut matcher = builder.build(text);
         // crop should return 10 last words with a marker at the start.

milli/src/search/new/mod.rs

@@ -530,11 +530,15 @@ fn resolve_sort_criteria<'ctx, Query: RankingRuleQueryTrait>(
     Ok(())
 }
 
-pub fn filtered_universe(ctx: &SearchContext, filters: &Option<Filter>) -> Result<RoaringBitmap> {
+pub fn filtered_universe(
+    index: &Index,
+    txn: &RoTxn<'_>,
+    filters: &Option<Filter>,
+) -> Result<RoaringBitmap> {
     Ok(if let Some(filters) = filters {
-        filters.evaluate(ctx.txn, ctx.index)?
+        filters.evaluate(txn, index)?
     } else {
-        ctx.index.documents_ids(ctx.txn)?
+        index.documents_ids(txn)?
     })
 }
 

milli/src/search/new/query_term/parse_query.rs

@@ -119,7 +119,7 @@ pub fn located_query_terms_from_tokens(
                             if let Some(located_query_term) = phrase.build(ctx) {
                                 // as we are evaluating a negative operator we put the phrase
                                 // in the negative one *but* we don't reset the negative operator
-                                // as we are immediately starting a new negative phrase.
+                                // as we are immediatly starting a new negative phrase.
                                 if negative_phrase {
                                     negative_phrases.push(located_query_term);
                                 } else {

milli/src/search/recommend.rs

@@ -0,0 +1,198 @@
+use std::sync::Arc;
+
+use ordered_float::OrderedFloat;
+use roaring::RoaringBitmap;
+use serde_json::Value;
+
+use crate::score_details::{self, ScoreDetails};
+use crate::vector::Embedder;
+use crate::{filtered_universe, DocumentId, Filter, Index, Result, SearchResult};
+
+enum RecommendKind<'a> {
+    Id(DocumentId),
+    Prompt { prompt: &'a str, context: Option<Value>, id: Option<DocumentId> },
+}
+
+pub struct Recommend<'a> {
+    kind: RecommendKind<'a>,
+    // this should be linked to the String in the query
+    filter: Option<Filter<'a>>,
+    offset: usize,
+    limit: usize,
+    rtxn: &'a heed::RoTxn<'a>,
+    index: &'a Index,
+    embedder_name: String,
+    embedder: Arc<Embedder>,
+}
+
+impl<'a> Recommend<'a> {
+    pub fn with_docid(
+        id: DocumentId,
+        offset: usize,
+        limit: usize,
+        index: &'a Index,
+        rtxn: &'a heed::RoTxn<'a>,
+        embedder_name: String,
+        embedder: Arc<Embedder>,
+    ) -> Self {
+        Self {
+            kind: RecommendKind::Id(id),
+            filter: None,
+            offset,
+            limit,
+            rtxn,
+            index,
+            embedder_name,
+            embedder,
+        }
+    }
+
+    pub fn with_prompt(
+        prompt: &'a str,
+        id: Option<DocumentId>,
+        context: Option<Value>,
+        offset: usize,
+        limit: usize,
+        index: &'a Index,
+        rtxn: &'a heed::RoTxn<'a>,
+        embedder_name: String,
+        embedder: Arc<Embedder>,
+    ) -> Self {
+        Self {
+            kind: RecommendKind::Prompt { prompt, context, id },
+            filter: None,
+            offset,
+            limit,
+            rtxn,
+            index,
+            embedder_name,
+            embedder,
+        }
+    }
+
+    pub fn filter(&mut self, filter: Filter<'a>) -> &mut Self {
+        self.filter = Some(filter);
+        self
+    }
+
+    pub fn execute(&self) -> Result<SearchResult> {
+        let universe = filtered_universe(self.index, self.rtxn, &self.filter)?;
+
+        let embedder_index =
+            self.index
+                .embedder_category_id
+                .get(self.rtxn, &self.embedder_name)?
+                .ok_or_else(|| crate::UserError::InvalidEmbedder(self.embedder_name.to_owned()))?;
+
+        let writer_index = (embedder_index as u16) << 8;
+        let readers: std::result::Result<Vec<_>, _> = (0..=u8::MAX)
+            .map_while(|k| {
+                arroy::Reader::open(self.rtxn, writer_index | (k as u16), self.index.vector_arroy)
+                    .map(Some)
+                    .or_else(|e| match e {
+                        arroy::Error::MissingMetadata => Ok(None),
+                        e => Err(e),
+                    })
+                    .transpose()
+            })
+            .collect();
+
+        let readers = readers?;
+
+        let mut results = Vec::new();
+
+        /// FIXME: make id optional...
+        let id = match &self.kind {
+            RecommendKind::Id(id) => *id,
+            RecommendKind::Prompt { prompt, context, id } => id.unwrap(),
+        };
+
+        let personalization_vector = if let RecommendKind::Prompt { prompt, context, id } =
+            &self.kind
+        {
+            let fields_ids_map = self.index.fields_ids_map(self.rtxn)?;
+
+            let document = if let Some(id) = id {
+                Some(self.index.iter_documents(self.rtxn, std::iter::once(*id))?.next().unwrap()?.1)
+            } else {
+                None
+            };
+            let document = document
+                .map(|document| crate::prompt::Document::from_doc_obkv(document, &fields_ids_map));
+
+            let context =
+                crate::prompt::recommend::Context::new(document.as_ref(), context.clone());
+
+            /// FIXME: handle error bad template
+            let template =
+                liquid::ParserBuilder::new().stdlib().build().unwrap().parse(prompt).unwrap();
+
+            /// FIXME: handle error bad context
+            let rendered = template.render(&context).unwrap();
+
+            /// FIXME: handle embedding error
+            Some(self.embedder.embed_one(rendered).unwrap())
+        } else {
+            None
+        };
+
+        for reader in readers.iter() {
+            let nns_by_item = reader.nns_by_item(
+                self.rtxn,
+                id,
+                self.limit + self.offset + 1,
+                None,
+                Some(&universe),
+            )?;
+
+            if let Some(nns_by_item) = nns_by_item {
+                let mut nns = match &personalization_vector {
+                    Some(vector) => {
+                        let candidates: RoaringBitmap =
+                            nns_by_item.iter().map(|(docid, _)| docid).collect();
+                        reader.nns_by_vector(
+                            self.rtxn,
+                            vector,
+                            self.limit + self.offset + 1,
+                            None,
+                            Some(&candidates),
+                        )?
+                    }
+                    None => nns_by_item,
+                };
+
+                results.append(&mut nns);
+            }
+        }
+
+        results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance));
+
+        let mut documents_ids = Vec::with_capacity(self.limit);
+        let mut document_scores = Vec::with_capacity(self.limit);
+
+        // skip offset +1 to skip the target document that is normally returned
+        for (docid, distance) in results.into_iter().skip(self.offset + 1) {
+            documents_ids.push(docid);
+
+            let score = 1.0 - distance;
+            let score = self
+                .embedder
+                .distribution()
+                .map(|distribution| distribution.shift(score))
+                .unwrap_or(score);
+
+            let score = ScoreDetails::Vector(score_details::Vector { similarity: Some(score) });
+
+            document_scores.push(vec![score]);
+        }
+
+        Ok(SearchResult {
+            matching_words: Default::default(),
+            candidates: universe,
+            documents_ids,
+            document_scores,
+            degraded: false,
+            used_negative_operator: false,
+        })
+    }
+}

milli/src/thread_pool_no_abort.rs

@@ -1,69 +0,0 @@
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-
-use rayon::{ThreadPool, ThreadPoolBuilder};
-use thiserror::Error;
-
-/// A rayon ThreadPool wrapper that can catch panics in the pool
-/// and modifies the install function accordingly.
-#[derive(Debug)]
-pub struct ThreadPoolNoAbort {
-    thread_pool: ThreadPool,
-    /// Set to true if the thread pool catched a panic.
-    pool_catched_panic: Arc<AtomicBool>,
-}
-
-impl ThreadPoolNoAbort {
-    pub fn install<OP, R>(&self, op: OP) -> Result<R, PanicCatched>
-    where
-        OP: FnOnce() -> R + Send,
-        R: Send,
-    {
-        let output = self.thread_pool.install(op);
-        // While reseting the pool panic catcher we return an error if we catched one.
-        if self.pool_catched_panic.swap(false, Ordering::SeqCst) {
-            Err(PanicCatched)
-        } else {
-            Ok(output)
-        }
-    }
-
-    pub fn current_num_threads(&self) -> usize {
-        self.thread_pool.current_num_threads()
-    }
-}
-
-#[derive(Error, Debug)]
-#[error("A panic occured. Read the logs to find more information about it")]
-pub struct PanicCatched;
-
-#[derive(Default)]
-pub struct ThreadPoolNoAbortBuilder(ThreadPoolBuilder);
-
-impl ThreadPoolNoAbortBuilder {
-    pub fn new() -> ThreadPoolNoAbortBuilder {
-        ThreadPoolNoAbortBuilder::default()
-    }
-
-    pub fn thread_name<F>(mut self, closure: F) -> Self
-    where
-        F: FnMut(usize) -> String + 'static,
-    {
-        self.0 = self.0.thread_name(closure);
-        self
-    }
-
-    pub fn num_threads(mut self, num_threads: usize) -> ThreadPoolNoAbortBuilder {
-        self.0 = self.0.num_threads(num_threads);
-        self
-    }
-
-    pub fn build(mut self) -> Result<ThreadPoolNoAbort, rayon::ThreadPoolBuildError> {
-        let pool_catched_panic = Arc::new(AtomicBool::new(false));
-        self.0 = self.0.panic_handler({
-            let catched_panic = pool_catched_panic.clone();
-            move |_result| catched_panic.store(true, Ordering::SeqCst)
-        });
-        Ok(ThreadPoolNoAbort { thread_pool: self.0.build()?, pool_catched_panic })
-    }
-}

milli/src/update/del_add.rs

@@ -71,8 +71,8 @@ pub enum DelAddOperation {
 /// putting each deletion obkv's keys under an DelAdd::Deletion
 /// and putting each addition obkv's keys under an DelAdd::Addition
 pub fn del_add_from_two_obkvs<K: obkv::Key + PartialOrd + Ord>(
-    deletion: &obkv::KvReader<K>,
-    addition: &obkv::KvReader<K>,
+    deletion: obkv::KvReader<K>,
+    addition: obkv::KvReader<K>,
     buffer: &mut Vec<u8>,
 ) -> Result<(), std::io::Error> {
     use itertools::merge_join_by;

milli/src/update/facet/incremental.rs

@@ -499,7 +499,7 @@ impl FacetsUpdateIncrementalInner {
                     ModificationResult::Expand | ModificationResult::Reduce { .. }
                 )
             {
-                // if any modification occurred, insert it in the database.
+                // if any modification occured, insert it in the database.
                 self.db.put(txn, &insertion_key.as_ref(), &updated_value)?;
                 Ok(insertion_key_modification)
             } else {

milli/src/update/index_documents/extract/extract_docid_word_positions.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::BufReader;
@@ -12,7 +12,6 @@ use serde_json::Value;
 use super::helpers::{create_sorter, keep_latest_obkv, sorter_into_reader, GrenadParameters};
 use crate::error::{InternalError, SerializationError};
 use crate::update::del_add::{del_add_from_two_obkvs, DelAdd, KvReaderDelAdd};
-use crate::update::settings::{InnerIndexSettings, InnerIndexSettingsDiff};
 use crate::{FieldId, Result, MAX_POSITION_PER_ATTRIBUTE, MAX_WORD_LENGTH};
 
 pub type ScriptLanguageDocidsMap = HashMap<(Script, Language), (RoaringBitmap, RoaringBitmap)>;
@@ -26,7 +25,10 @@ pub type ScriptLanguageDocidsMap = HashMap<(Script, Language), (RoaringBitmap, R
 pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    settings_diff: &InnerIndexSettingsDiff,
+    searchable_fields: &Option<HashSet<FieldId>>,
+    stop_words: Option<&fst::Set<Vec<u8>>>,
+    allowed_separators: Option<&[&str]>,
+    dictionary: Option<&[&str]>,
     max_positions_per_attributes: Option<u32>,
 ) -> Result<(grenad::Reader<BufReader<File>>, ScriptLanguageDocidsMap)> {
     puffin::profile_function!();
@@ -34,7 +36,6 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     let max_positions_per_attributes = max_positions_per_attributes
         .map_or(MAX_POSITION_PER_ATTRIBUTE, |max| max.min(MAX_POSITION_PER_ATTRIBUTE));
     let max_memory = indexer.max_memory_by_thread();
-    let force_reindexing = settings_diff.reindex_searchable();
 
     // initialize destination values.
     let mut documents_ids = RoaringBitmap::new();
@@ -55,37 +56,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     let mut value_buffer = Vec::new();
 
     // initialize tokenizer.
-    let old_stop_words = settings_diff.old.stop_words.as_ref();
-    let old_separators: Option<Vec<_>> = settings_diff
-        .old
-        .allowed_separators
-        .as_ref()
-        .map(|s| s.iter().map(String::as_str).collect());
-    let old_dictionary: Option<Vec<_>> =
-        settings_diff.old.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
-    let mut del_builder = tokenizer_builder(
-        old_stop_words,
-        old_separators.as_deref(),
-        old_dictionary.as_deref(),
-        None,
-    );
-    let del_tokenizer = del_builder.build();
-
-    let new_stop_words = settings_diff.new.stop_words.as_ref();
-    let new_separators: Option<Vec<_>> = settings_diff
-        .new
-        .allowed_separators
-        .as_ref()
-        .map(|s| s.iter().map(String::as_str).collect());
-    let new_dictionary: Option<Vec<_>> =
-        settings_diff.new.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
-    let mut add_builder = tokenizer_builder(
-        new_stop_words,
-        new_separators.as_deref(),
-        new_dictionary.as_deref(),
-        None,
-    );
-    let add_tokenizer = add_builder.build();
+    let mut builder = tokenizer_builder(stop_words, allowed_separators, dictionary, None);
+    let tokenizer = builder.build();
 
     // iterate over documents.
     let mut cursor = obkv_documents.into_cursor()?;
@@ -97,7 +69,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
         let obkv = KvReader::<FieldId>::new(value);
 
         // if the searchable fields didn't change, skip the searchable indexing for this document.
-        if !force_reindexing && !searchable_fields_changed(&obkv, settings_diff) {
+        if !searchable_fields_changed(&KvReader::<FieldId>::new(value), searchable_fields) {
             continue;
         }
 
@@ -113,8 +85,11 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
                 // deletions
                 lang_safe_tokens_from_document(
                     &obkv,
-                    &settings_diff.old,
-                    &del_tokenizer,
+                    searchable_fields,
+                    &tokenizer,
+                    stop_words,
+                    allowed_separators,
+                    dictionary,
                     max_positions_per_attributes,
                     DelAdd::Deletion,
                     &mut del_buffers,
@@ -124,8 +99,11 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
                 // additions
                 lang_safe_tokens_from_document(
                     &obkv,
-                    &settings_diff.new,
-                    &add_tokenizer,
+                    searchable_fields,
+                    &tokenizer,
+                    stop_words,
+                    allowed_separators,
+                    dictionary,
                     max_positions_per_attributes,
                     DelAdd::Addition,
                     &mut add_buffers,
@@ -140,8 +118,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
         // transforming two KV<FieldId, KV<u16, String>> into one KV<FieldId, KV<DelAdd, KV<u16, String>>>
         value_buffer.clear();
         del_add_from_two_obkvs(
-            &KvReader::<FieldId>::new(del_obkv),
-            &KvReader::<FieldId>::new(add_obkv),
+            KvReader::<FieldId>::new(del_obkv),
+            KvReader::<FieldId>::new(add_obkv),
             &mut value_buffer,
         )?;
 
@@ -182,9 +160,8 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
 /// Check if any searchable fields of a document changed.
 fn searchable_fields_changed(
     obkv: &KvReader<FieldId>,
-    settings_diff: &InnerIndexSettingsDiff,
+    searchable_fields: &Option<HashSet<FieldId>>,
 ) -> bool {
-    let searchable_fields = &settings_diff.new.searchable_fields_ids;
     for (field_id, field_bytes) in obkv.iter() {
         if searchable_fields.as_ref().map_or(true, |sf| sf.contains(&field_id)) {
             let del_add = KvReaderDelAdd::new(field_bytes);
@@ -229,10 +206,14 @@ fn tokenizer_builder<'a>(
 
 /// Extract words mapped with their positions of a document,
 /// ensuring no Language detection mistakes was made.
+#[allow(clippy::too_many_arguments)] // FIXME: consider grouping arguments in a struct
 fn lang_safe_tokens_from_document<'a>(
     obkv: &KvReader<FieldId>,
-    settings: &InnerIndexSettings,
+    searchable_fields: &Option<HashSet<FieldId>>,
     tokenizer: &Tokenizer,
+    stop_words: Option<&fst::Set<Vec<u8>>>,
+    allowed_separators: Option<&[&str]>,
+    dictionary: Option<&[&str]>,
     max_positions_per_attributes: u32,
     del_add: DelAdd,
     buffers: &'a mut Buffers,
@@ -241,7 +222,7 @@ fn lang_safe_tokens_from_document<'a>(
 
     tokens_from_document(
         obkv,
-        &settings.searchable_fields_ids,
+        searchable_fields,
         tokenizer,
         max_positions_per_attributes,
         del_add,
@@ -265,15 +246,12 @@ fn lang_safe_tokens_from_document<'a>(
         // then we don't rerun the extraction.
         if !script_language.is_empty() {
             // build a new temporary tokenizer including the allow list.
-            let stop_words = settings.stop_words.as_ref();
-            let separators: Option<Vec<_>> = settings
-                .allowed_separators
-                .as_ref()
-                .map(|s| s.iter().map(String::as_str).collect());
-            let dictionary: Option<Vec<_>> =
-                settings.dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
-            let mut builder =
-                tokenizer_builder(stop_words, separators.as_deref(), dictionary.as_deref(), None);
+            let mut builder = tokenizer_builder(
+                stop_words,
+                allowed_separators,
+                dictionary,
+                Some(&script_language),
+            );
             let tokenizer = builder.build();
 
             script_language_word_count.clear();
@@ -281,7 +259,7 @@ fn lang_safe_tokens_from_document<'a>(
             // rerun the extraction.
             tokens_from_document(
                 obkv,
-                &settings.searchable_fields_ids,
+                searchable_fields,
                 &tokenizer,
                 max_positions_per_attributes,
                 del_add,
@@ -298,7 +276,7 @@ fn lang_safe_tokens_from_document<'a>(
 /// Extract words mapped with their positions of a document.
 fn tokens_from_document<'a>(
     obkv: &KvReader<FieldId>,
-    searchable_fields: &Option<Vec<FieldId>>,
+    searchable_fields: &Option<HashSet<FieldId>>,
     tokenizer: &Tokenizer,
     max_positions_per_attributes: u32,
     del_add: DelAdd,

milli/src/update/index_documents/extract/extract_facet_number_docids.rs

@@ -10,7 +10,6 @@ use crate::heed_codec::facet::{
     FacetGroupKey, FacetGroupKeyCodec, FieldDocIdFacetF64Codec, OrderedF64Codec,
 };
 use crate::update::del_add::{KvReaderDelAdd, KvWriterDelAdd};
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::Result;
 
 /// Extracts the facet number and the documents ids where this facet number appear.
@@ -21,7 +20,6 @@ use crate::Result;
 pub fn extract_facet_number_docids<R: io::Read + io::Seek>(
     fid_docid_facet_number: grenad::Reader<R>,
     indexer: GrenadParameters,
-    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_facet_string_docids.rs

@@ -15,7 +15,6 @@ use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::index_documents::helpers::{
     merge_deladd_btreeset_string, merge_deladd_cbo_roaring_bitmaps,
 };
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 
 /// Extracts the facet string and the documents ids where this facet string appear.
@@ -26,7 +25,6 @@ use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
     docid_fid_facet_string: grenad::Reader<R>,
     indexer: GrenadParameters,
-    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::{self, BufReader};
@@ -20,7 +20,6 @@ use crate::error::InternalError;
 use crate::facet::value_encoding::f64_into_bytes;
 use crate::update::del_add::{DelAdd, KvWriterDelAdd};
 use crate::update::index_documents::{create_writer, writer_into_reader};
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{CboRoaringBitmapCodec, DocumentId, Error, FieldId, Result, MAX_FACET_VALUE_LENGTH};
 
 /// The length of the elements that are always in the buffer when inserting new values.
@@ -37,14 +36,14 @@ pub struct ExtractedFacetValues {
 
 /// Extracts the facet values of each faceted field of each document.
 ///
-/// Returns the generated grenad reader containing the docid the fid and the original value as key
+/// Returns the generated grenad reader containing the docid the fid and the orginal value as key
 /// and the normalized value as value extracted from the given chunk of documents.
 /// We need the fid of the geofields to correctly parse them as numbers if they were sent as strings initially.
 #[tracing::instrument(level = "trace", skip_all, target = "indexing::extract")]
 pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    settings_diff: &InnerIndexSettingsDiff,
+    faceted_fields: &HashSet<FieldId>,
     geo_fields_ids: Option<(FieldId, FieldId)>,
 ) -> Result<ExtractedFacetValues> {
     puffin::profile_function!();
@@ -83,9 +82,7 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
         let obkv = obkv::KvReader::new(value);
 
         for (field_id, field_bytes) in obkv.iter() {
-            let delete_faceted = settings_diff.old.faceted_fields_ids.contains(&field_id);
-            let add_faceted = settings_diff.new.faceted_fields_ids.contains(&field_id);
-            if delete_faceted || add_faceted {
+            if faceted_fields.contains(&field_id) {
                 numbers_key_buffer.clear();
                 strings_key_buffer.clear();
 
@@ -102,12 +99,11 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
                 strings_key_buffer.extend_from_slice(docid_bytes);
 
                 let del_add_obkv = obkv::KvReader::new(field_bytes);
-                let del_value = match del_add_obkv.get(DelAdd::Deletion).filter(|_| delete_faceted)
-                {
+                let del_value = match del_add_obkv.get(DelAdd::Deletion) {
                     Some(bytes) => Some(from_slice(bytes).map_err(InternalError::SerdeJson)?),
                     None => None,
                 };
-                let add_value = match del_add_obkv.get(DelAdd::Addition).filter(|_| add_faceted) {
+                let add_value = match del_add_obkv.get(DelAdd::Addition) {
                     Some(bytes) => Some(from_slice(bytes).map_err(InternalError::SerdeJson)?),
                     None => None,
                 };

milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs

@@ -10,7 +10,6 @@ use super::helpers::{
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::Result;
 
 const MAX_COUNTED_WORDS: usize = 30;
@@ -24,7 +23,6 @@ const MAX_COUNTED_WORDS: usize = 30;
 pub fn extract_fid_word_count_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -17,9 +17,8 @@ use crate::error::UserError;
 use crate::prompt::Prompt;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
 use crate::update::index_documents::helpers::try_split_at;
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::vector::Embedder;
-use crate::{DocumentId, InternalError, Result, ThreadPoolNoAbort, VectorOrArrayOfVectors};
+use crate::{DocumentId, FieldsIdsMap, InternalError, Result, VectorOrArrayOfVectors};
 
 /// The length of the elements that are always in the buffer when inserting new values.
 const TRUNCATE_SIZE: usize = size_of::<DocumentId>();
@@ -72,15 +71,12 @@ impl VectorStateDelta {
 pub fn extract_vector_points<R: io::Read + io::Seek>(
     obkv_documents: grenad::Reader<R>,
     indexer: GrenadParameters,
-    settings_diff: &InnerIndexSettingsDiff,
+    field_id_map: &FieldsIdsMap,
     prompt: &Prompt,
     embedder_name: &str,
 ) -> Result<ExtractedVectorPoints> {
     puffin::profile_function!();
 
-    let old_fields_ids_map = &settings_diff.old.fields_ids_map;
-    let new_fields_ids_map = &settings_diff.new.fields_ids_map;
-
     // (docid, _index) -> KvWriterDelAdd -> Vector
     let mut manual_vectors_writer = create_writer(
         indexer.chunk_compression_type,
@@ -102,6 +98,8 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
         tempfile::tempfile()?,
     );
 
+    let vectors_fid = field_id_map.id("_vectors");
+
     let mut key_buffer = Vec::new();
     let mut cursor = obkv_documents.into_cursor()?;
     while let Some((key, value)) = cursor.move_on_next()? {
@@ -118,29 +116,15 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
         // lazily get it when needed
         let document_id = || -> Value { from_utf8(external_id_bytes).unwrap().into() };
 
-        // the vector field id may have changed
-        let old_vectors_fid = old_fields_ids_map.id("_vectors");
-        // filter the old vector fid if the settings has been changed forcing reindexing.
-        let old_vectors_fid = old_vectors_fid.filter(|_| !settings_diff.reindex_vectors());
+        let vectors_field = vectors_fid
+            .and_then(|vectors_fid| obkv.get(vectors_fid))
+            .map(KvReaderDelAdd::new)
+            .map(|obkv| to_vector_maps(obkv, document_id))
+            .transpose()?;
 
-        let new_vectors_fid = new_fields_ids_map.id("_vectors");
-        let vectors_field = {
-            let del = old_vectors_fid
-                .and_then(|vectors_fid| obkv.get(vectors_fid))
-                .map(KvReaderDelAdd::new)
-                .map(|obkv| to_vector_map(obkv, DelAdd::Deletion, &document_id))
-                .transpose()?
-                .flatten();
-            let add = new_vectors_fid
-                .and_then(|vectors_fid| obkv.get(vectors_fid))
-                .map(KvReaderDelAdd::new)
-                .map(|obkv| to_vector_map(obkv, DelAdd::Addition, &document_id))
-                .transpose()?
-                .flatten();
-            (del, add)
-        };
-
-        let (del_map, add_map) = vectors_field;
+        let (del_map, add_map) = vectors_field.unzip();
+        let del_map = del_map.flatten();
+        let add_map = add_map.flatten();
 
         let del_value = del_map.and_then(|mut map| map.remove(embedder_name));
         let add_value = add_map.and_then(|mut map| map.remove(embedder_name));
@@ -171,7 +155,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
                     VectorStateDelta::NowGenerated(prompt.render(
                         obkv,
                         DelAdd::Addition,
-                        new_fields_ids_map,
+                        field_id_map,
                     )?)
                 } else {
                     VectorStateDelta::NowRemoved
@@ -198,16 +182,10 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
 
                 if document_is_kept {
                     // Don't give up if the old prompt was failing
-                    let old_prompt = Some(prompt)
-                        // TODO: this filter works because we erase the vec database when a embedding setting changes.
-                        // When vector pipeline will be optimized, this should be removed.
-                        .filter(|_| !settings_diff.reindex_vectors())
-                        .map(|p| {
-                            p.render(obkv, DelAdd::Deletion, old_fields_ids_map).unwrap_or_default()
-                        });
-                    let new_prompt = prompt.render(obkv, DelAdd::Addition, new_fields_ids_map)?;
-                    if old_prompt.as_ref() != Some(&new_prompt) {
-                        let old_prompt = old_prompt.unwrap_or_default();
+                    let old_prompt =
+                        prompt.render(obkv, DelAdd::Deletion, field_id_map).unwrap_or_default();
+                    let new_prompt = prompt.render(obkv, DelAdd::Addition, field_id_map)?;
+                    if old_prompt != new_prompt {
                         tracing::trace!(
                             "🚀 Changing prompt from\n{old_prompt}\n===to===\n{new_prompt}"
                         );
@@ -229,7 +207,6 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
             &mut manual_vectors_writer,
             &mut key_buffer,
             delta,
-            settings_diff,
         )?;
     }
 
@@ -243,6 +220,15 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     })
 }
 
+fn to_vector_maps(
+    obkv: KvReaderDelAdd,
+    document_id: impl Fn() -> Value,
+) -> Result<(Option<serde_json::Map<String, Value>>, Option<serde_json::Map<String, Value>>)> {
+    let del = to_vector_map(obkv, DelAdd::Deletion, &document_id)?;
+    let add = to_vector_map(obkv, DelAdd::Addition, &document_id)?;
+    Ok((del, add))
+}
+
 fn to_vector_map(
     obkv: KvReaderDelAdd,
     side: DelAdd,
@@ -270,15 +256,10 @@ fn push_vectors_diff(
     manual_vectors_writer: &mut Writer<BufWriter<File>>,
     key_buffer: &mut Vec<u8>,
     delta: VectorStateDelta,
-    settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<()> {
     puffin::profile_function!();
     let (must_remove, prompt, (mut del_vectors, mut add_vectors)) = delta.into_values();
-    if must_remove
-    // TODO: the below condition works because we erase the vec database when a embedding setting changes.
-    // When vector pipeline will be optimized, this should be removed.
-    && !settings_diff.reindex_vectors()
-    {
+    if must_remove {
         key_buffer.truncate(TRUNCATE_SIZE);
         remove_vectors_writer.insert(&key_buffer, [])?;
     }
@@ -306,16 +287,12 @@ fn push_vectors_diff(
         match eob {
             EitherOrBoth::Both(_, _) => (), // no need to touch anything
             EitherOrBoth::Left(vector) => {
-                // TODO: the below condition works because we erase the vec database when a embedding setting changes.
-                // When vector pipeline will be optimized, this should be removed.
-                if !settings_diff.reindex_vectors() {
-                    // We insert only the Del part of the Obkv to inform
-                    // that we only want to remove all those vectors.
-                    let mut obkv = KvWriterDelAdd::memory();
-                    obkv.insert(DelAdd::Deletion, cast_slice(&vector))?;
-                    let bytes = obkv.into_inner()?;
-                    manual_vectors_writer.insert(&key_buffer, bytes)?;
-                }
+                // We insert only the Del part of the Obkv to inform
+                // that we only want to remove all those vectors.
+                let mut obkv = KvWriterDelAdd::memory();
+                obkv.insert(DelAdd::Deletion, cast_slice(&vector))?;
+                let bytes = obkv.into_inner()?;
+                manual_vectors_writer.insert(&key_buffer, bytes)?;
             }
             EitherOrBoth::Right(vector) => {
                 // We insert only the Add part of the Obkv to inform
@@ -362,7 +339,7 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
     prompt_reader: grenad::Reader<R>,
     indexer: GrenadParameters,
     embedder: Arc<Embedder>,
-    request_threads: &ThreadPoolNoAbort,
+    request_threads: &rayon::ThreadPool,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
     let n_chunks = embedder.chunk_count_hint(); // chunk level parallelism

milli/src/update/index_documents/extract/extract_word_docids.rs

@@ -1,23 +1,20 @@
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashSet};
 use std::fs::File;
 use std::io::{self, BufReader};
 
-use heed::{BytesDecode, BytesEncode};
+use heed::BytesDecode;
 use obkv::KvReaderU16;
-use roaring::RoaringBitmap;
 
 use super::helpers::{
-    create_sorter, create_writer, merge_deladd_cbo_roaring_bitmaps, try_split_array_at,
-    writer_into_reader, GrenadParameters,
+    create_sorter, create_writer, merge_deladd_cbo_roaring_bitmaps, sorter_into_reader,
+    try_split_array_at, writer_into_reader, GrenadParameters,
 };
 use crate::error::SerializationError;
 use crate::heed_codec::StrBEU16Codec;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{is_noop_del_add_obkv, DelAdd, KvReaderDelAdd, KvWriterDelAdd};
-use crate::update::index_documents::helpers::sorter_into_reader;
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::update::MergeFn;
-use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result};
+use crate::{DocumentId, FieldId, Result};
 
 /// Extracts the word and the documents ids where this word appear.
 ///
@@ -30,7 +27,7 @@ use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result};
 pub fn extract_word_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-    settings_diff: &InnerIndexSettingsDiff,
+    exact_attributes: &HashSet<FieldId>,
 ) -> Result<(
     grenad::Reader<BufReader<File>>,
     grenad::Reader<BufReader<File>>,
@@ -46,7 +43,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|m| m / 3),
+        max_memory.map(|x| x / 3),
     );
     let mut key_buffer = Vec::new();
     let mut del_words = BTreeSet::new();
@@ -88,19 +85,13 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         add_words.clear();
     }
 
-    let mut word_fid_docids_writer = create_writer(
-        indexer.chunk_compression_type,
-        indexer.chunk_compression_level,
-        tempfile::tempfile()?,
-    );
-
     let mut word_docids_sorter = create_sorter(
         grenad::SortAlgorithm::Unstable,
         merge_deladd_cbo_roaring_bitmaps,
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|m| m / 3),
+        max_memory.map(|x| x / 3),
     );
 
     let mut exact_word_docids_sorter = create_sorter(
@@ -109,45 +100,31 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
         indexer.max_nb_chunks,
-        max_memory.map(|m| m / 3),
+        max_memory.map(|x| x / 3),
+    );
+
+    let mut word_fid_docids_writer = create_writer(
+        indexer.chunk_compression_type,
+        indexer.chunk_compression_level,
+        tempfile::tempfile()?,
     );
 
     let mut iter = word_fid_docids_sorter.into_stream_merger_iter()?;
-    let mut buffer = Vec::new();
-    // NOTE: replacing sorters by bitmap merging is less efficient, so, use sorters.
+    // TODO: replace sorters by writers by accumulating values into a buffer before inserting them.
     while let Some((key, value)) = iter.next()? {
         // only keep the value if their is a change to apply in the DB.
         if !is_noop_del_add_obkv(KvReaderDelAdd::new(value)) {
             word_fid_docids_writer.insert(key, value)?;
         }
 
-        let (w, fid) = StrBEU16Codec::bytes_decode(key)
+        let (word, fid) = StrBEU16Codec::bytes_decode(key)
             .map_err(|_| SerializationError::Decoding { db_name: Some(DOCID_WORD_POSITIONS) })?;
 
-        // merge all deletions
-        let obkv = KvReaderDelAdd::new(value);
-        if let Some(value) = obkv.get(DelAdd::Deletion) {
-            let delete_from_exact = settings_diff.old.exact_attributes.contains(&fid);
-            buffer.clear();
-            let mut obkv = KvWriterDelAdd::new(&mut buffer);
-            obkv.insert(DelAdd::Deletion, value)?;
-            if delete_from_exact {
-                exact_word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
-            } else {
-                word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
-            }
-        }
-        // merge all additions
-        if let Some(value) = obkv.get(DelAdd::Addition) {
-            let add_in_exact = settings_diff.new.exact_attributes.contains(&fid);
-            buffer.clear();
-            let mut obkv = KvWriterDelAdd::new(&mut buffer);
-            obkv.insert(DelAdd::Addition, value)?;
-            if add_in_exact {
-                exact_word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
-            } else {
-                word_docids_sorter.insert(w, obkv.into_inner().unwrap())?;
-            }
+        // every words contained in an attribute set to exact must be pushed in the exact_words list.
+        if exact_attributes.contains(&fid) {
+            exact_word_docids_sorter.insert(word.as_bytes(), value)?;
+        } else {
+            word_docids_sorter.insert(word.as_bytes(), value)?;
         }
     }
 
@@ -201,45 +178,3 @@ fn words_into_sorter(
 
     Ok(())
 }
-
-#[tracing::instrument(level = "trace", skip_all, target = "indexing::extract")]
-fn docids_into_writers<W>(
-    word: &str,
-    deletions: &RoaringBitmap,
-    additions: &RoaringBitmap,
-    writer: &mut grenad::Writer<W>,
-) -> Result<()>
-where
-    W: std::io::Write,
-{
-    if deletions == additions {
-        // if the same value is deleted and added, do nothing.
-        return Ok(());
-    }
-
-    // Write each value in the same KvDelAdd before inserting it in the final writer.
-    let mut obkv = KvWriterDelAdd::memory();
-    // deletions:
-    if !deletions.is_empty() && !deletions.is_subset(additions) {
-        obkv.insert(
-            DelAdd::Deletion,
-            CboRoaringBitmapCodec::bytes_encode(deletions).map_err(|_| {
-                SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) }
-            })?,
-        )?;
-    }
-    // additions:
-    if !additions.is_empty() {
-        obkv.insert(
-            DelAdd::Addition,
-            CboRoaringBitmapCodec::bytes_encode(additions).map_err(|_| {
-                SerializationError::Encoding { db_name: Some(DOCID_WORD_POSITIONS) }
-            })?,
-        )?;
-    }
-
-    // insert everything in the same writer.
-    writer.insert(word.as_bytes(), obkv.into_inner().unwrap())?;
-
-    Ok(())
-}

milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs

@@ -11,9 +11,8 @@ use super::helpers::{
 };
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
-use crate::proximity::{index_proximity, ProximityPrecision, MAX_DISTANCE};
+use crate::proximity::{index_proximity, MAX_DISTANCE};
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::{DocumentId, Result};
 
 /// Extracts the best proximity between pairs of words and the documents ids where this pair appear.
@@ -24,21 +23,8 @@ use crate::{DocumentId, Result};
 pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-    settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
-    let any_deletion = settings_diff.old.proximity_precision == ProximityPrecision::ByWord;
-    let any_addition = settings_diff.new.proximity_precision == ProximityPrecision::ByWord;
-
-    // early return if the data shouldn't be deleted nor created.
-    if !any_deletion && !any_addition {
-        let writer = create_writer(
-            indexer.chunk_compression_type,
-            indexer.chunk_compression_level,
-            tempfile::tempfile()?,
-        );
-        return writer_into_reader(writer);
-    }
 
     let max_memory = indexer.max_memory_by_thread();
 
@@ -91,10 +77,6 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
 
         let (del, add): (Result<_>, Result<_>) = rayon::join(
             || {
-                if !any_deletion {
-                    return Ok(());
-                }
-
                 // deletions
                 if let Some(deletion) = KvReaderDelAdd::new(value).get(DelAdd::Deletion) {
                     for (position, word) in KvReaderU16::new(deletion).iter() {
@@ -124,10 +106,6 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
                 Ok(())
             },
             || {
-                if !any_addition {
-                    return Ok(());
-                }
-
                 // additions
                 if let Some(addition) = KvReaderDelAdd::new(value).get(DelAdd::Addition) {
                     for (position, word) in KvReaderU16::new(addition).iter() {

milli/src/update/index_documents/extract/extract_word_position_docids.rs

@@ -11,7 +11,6 @@ use super::helpers::{
 use crate::error::SerializationError;
 use crate::index::db_name::DOCID_WORD_POSITIONS;
 use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
-use crate::update::settings::InnerIndexSettingsDiff;
 use crate::update::MergeFn;
 use crate::{bucketed_position, DocumentId, Result};
 
@@ -23,7 +22,6 @@ use crate::{bucketed_position, DocumentId, Result};
 pub fn extract_word_position_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-    _settings_diff: &InnerIndexSettingsDiff,
 ) -> Result<grenad::Reader<BufReader<File>>> {
     puffin::profile_function!();
 

milli/src/update/index_documents/extract/mod.rs

@@ -9,9 +9,9 @@ mod extract_word_docids;
 mod extract_word_pair_proximity_docids;
 mod extract_word_position_docids;
 
+use std::collections::HashSet;
 use std::fs::File;
 use std::io::BufReader;
-use std::sync::Arc;
 
 use crossbeam_channel::Sender;
 use rayon::prelude::*;
@@ -30,8 +30,9 @@ use self::extract_word_pair_proximity_docids::extract_word_pair_proximity_docids
 use self::extract_word_position_docids::extract_word_position_docids;
 use super::helpers::{as_cloneable_grenad, CursorClonableMmap, GrenadParameters};
 use super::{helpers, TypedChunk};
-use crate::update::settings::InnerIndexSettingsDiff;
-use crate::{FieldId, Result, ThreadPoolNoAbortBuilder};
+use crate::proximity::ProximityPrecision;
+use crate::vector::EmbeddingConfigs;
+use crate::{FieldId, FieldsIdsMap, Result};
 
 /// Extract data for each databases from obkv documents in parallel.
 /// Send data in grenad file over provided Sender.
@@ -42,10 +43,18 @@ pub(crate) fn data_from_obkv_documents(
     flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
+    searchable_fields: Option<HashSet<FieldId>>,
+    faceted_fields: HashSet<FieldId>,
     primary_key_id: FieldId,
     geo_fields_ids: Option<(FieldId, FieldId)>,
-    settings_diff: Arc<InnerIndexSettingsDiff>,
+    field_id_map: FieldsIdsMap,
+    stop_words: Option<fst::Set<Vec<u8>>>,
+    allowed_separators: Option<&[&str]>,
+    dictionary: Option<&[&str]>,
     max_positions_per_attributes: Option<u32>,
+    exact_attributes: HashSet<FieldId>,
+    proximity_precision: ProximityPrecision,
+    embedders: EmbeddingConfigs,
 ) -> Result<()> {
     puffin::profile_function!();
 
@@ -58,7 +67,8 @@ pub(crate) fn data_from_obkv_documents(
                         original_documents_chunk,
                         indexer,
                         lmdb_writer_sx.clone(),
-                        settings_diff.clone(),
+                        field_id_map.clone(),
+                        embedders.clone(),
                     )
                 })
                 .collect::<Result<()>>()
@@ -71,9 +81,13 @@ pub(crate) fn data_from_obkv_documents(
                         flattened_obkv_chunks,
                         indexer,
                         lmdb_writer_sx.clone(),
+                        &searchable_fields,
+                        &faceted_fields,
                         primary_key_id,
                         geo_fields_ids,
-                        settings_diff.clone(),
+                        &stop_words,
+                        &allowed_separators,
+                        &dictionary,
                         max_positions_per_attributes,
                     )
                 })
@@ -86,12 +100,13 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             docid_word_positions_chunk.clone(),
                             indexer,
-                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_fid_word_count_docids,
                             TypedChunk::FieldIdWordCountDocids,
                             "field-id-wordcount-docids",
                         );
+
+                        let exact_attributes = exact_attributes.clone();
                         run_extraction_task::<
                             _,
                             _,
@@ -103,9 +118,10 @@ pub(crate) fn data_from_obkv_documents(
                         >(
                             docid_word_positions_chunk.clone(),
                             indexer,
-                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
-                            extract_word_docids,
+                            move |doc_word_pos, indexer| {
+                                extract_word_docids(doc_word_pos, indexer, &exact_attributes)
+                            },
                             |(
                                 word_docids_reader,
                                 exact_word_docids_reader,
@@ -123,7 +139,6 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             docid_word_positions_chunk.clone(),
                             indexer,
-                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_word_position_docids,
                             TypedChunk::WordPositionDocids,
@@ -137,7 +152,6 @@ pub(crate) fn data_from_obkv_documents(
                         >(
                             fid_docid_facet_strings_chunk.clone(),
                             indexer,
-                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_facet_string_docids,
                             TypedChunk::FieldIdFacetStringDocids,
@@ -147,22 +161,22 @@ pub(crate) fn data_from_obkv_documents(
                         run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
                             fid_docid_facet_numbers_chunk.clone(),
                             indexer,
-                            settings_diff.clone(),
                             lmdb_writer_sx.clone(),
                             extract_facet_number_docids,
                             TypedChunk::FieldIdFacetNumberDocids,
                             "field-id-facet-number-docids",
                         );
 
-                        run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
-                            docid_word_positions_chunk.clone(),
-                            indexer,
-                            settings_diff.clone(),
-                            lmdb_writer_sx.clone(),
-                            extract_word_pair_proximity_docids,
-                            TypedChunk::WordPairProximityDocids,
-                            "word-pair-proximity-docids",
-                        );
+                        if proximity_precision == ProximityPrecision::ByWord {
+                            run_extraction_task::<_, _, grenad::Reader<BufReader<File>>>(
+                                docid_word_positions_chunk.clone(),
+                                indexer,
+                                lmdb_writer_sx.clone(),
+                                extract_word_pair_proximity_docids,
+                                TypedChunk::WordPairProximityDocids,
+                                "word-pair-proximity-docids",
+                            );
+                        }
                     }
 
                     Ok(())
@@ -181,17 +195,12 @@ pub(crate) fn data_from_obkv_documents(
 fn run_extraction_task<FE, FS, M>(
     chunk: grenad::Reader<CursorClonableMmap>,
     indexer: GrenadParameters,
-    settings_diff: Arc<InnerIndexSettingsDiff>,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     extract_fn: FE,
     serialize_fn: FS,
     name: &'static str,
 ) where
-    FE: Fn(
-            grenad::Reader<CursorClonableMmap>,
-            GrenadParameters,
-            &InnerIndexSettingsDiff,
-        ) -> Result<M>
+    FE: Fn(grenad::Reader<CursorClonableMmap>, GrenadParameters) -> Result<M>
         + Sync
         + Send
         + 'static,
@@ -204,7 +213,7 @@ fn run_extraction_task<FE, FS, M>(
         let child_span = tracing::trace_span!(target: "indexing::extract::details", parent: &current_span, "extract_multiple_chunks");
         let _entered = child_span.enter();
         puffin::profile_scope!("extract_multiple_chunks", name);
-        match extract_fn(chunk, indexer, &settings_diff) {
+        match extract_fn(chunk, indexer) {
             Ok(chunk) => {
                 let _ = lmdb_writer_sx.send(Ok(serialize_fn(chunk)));
             }
@@ -221,7 +230,8 @@ fn send_original_documents_data(
     original_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
-    settings_diff: Arc<InnerIndexSettingsDiff>,
+    field_id_map: FieldsIdsMap,
+    embedders: EmbeddingConfigs,
 ) -> Result<()> {
     let original_documents_chunk =
         original_documents_chunk.and_then(|c| unsafe { as_cloneable_grenad(&c) })?;
@@ -229,58 +239,55 @@ fn send_original_documents_data(
     let documents_chunk_cloned = original_documents_chunk.clone();
     let lmdb_writer_sx_cloned = lmdb_writer_sx.clone();
 
-    let request_threads = ThreadPoolNoAbortBuilder::new()
+    let request_threads = rayon::ThreadPoolBuilder::new()
         .num_threads(crate::vector::REQUEST_PARALLELISM)
         .thread_name(|index| format!("embedding-request-{index}"))
         .build()?;
 
-    if settings_diff.reindex_vectors() || !settings_diff.settings_update_only() {
-        let settings_diff = settings_diff.clone();
-        rayon::spawn(move || {
-            for (name, (embedder, prompt)) in settings_diff.new.embedding_configs.clone() {
-                let result = extract_vector_points(
-                    documents_chunk_cloned.clone(),
-                    indexer,
-                    &settings_diff,
-                    &prompt,
-                    &name,
-                );
-                match result {
-                    Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
-                        let embeddings = match extract_embeddings(
-                            prompts,
-                            indexer,
-                            embedder.clone(),
-                            &request_threads,
-                        ) {
-                            Ok(results) => Some(results),
-                            Err(error) => {
-                                let _ = lmdb_writer_sx_cloned.send(Err(error));
-                                None
-                            }
-                        };
-
-                        if !(remove_vectors.is_empty()
-                            && manual_vectors.is_empty()
-                            && embeddings.as_ref().map_or(true, |e| e.is_empty()))
-                        {
-                            let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
-                                remove_vectors,
-                                embeddings,
-                                expected_dimension: embedder.dimensions(),
-                                manual_vectors,
-                                embedder_name: name,
-                            }));
+    rayon::spawn(move || {
+        for (name, (embedder, prompt)) in embedders {
+            let result = extract_vector_points(
+                documents_chunk_cloned.clone(),
+                indexer,
+                &field_id_map,
+                &prompt,
+                &name,
+            );
+            match result {
+                Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
+                    let embeddings = match extract_embeddings(
+                        prompts,
+                        indexer,
+                        embedder.clone(),
+                        &request_threads,
+                    ) {
+                        Ok(results) => Some(results),
+                        Err(error) => {
+                            let _ = lmdb_writer_sx_cloned.send(Err(error));
+                            None
                         }
-                    }
+                    };
 
-                    Err(error) => {
-                        let _ = lmdb_writer_sx_cloned.send(Err(error));
+                    if !(remove_vectors.is_empty()
+                        && manual_vectors.is_empty()
+                        && embeddings.as_ref().map_or(true, |e| e.is_empty()))
+                    {
+                        let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
+                            remove_vectors,
+                            embeddings,
+                            expected_dimension: embedder.dimensions(),
+                            manual_vectors,
+                            embedder_name: name,
+                        }));
                     }
                 }
+
+                Err(error) => {
+                    let _ = lmdb_writer_sx_cloned.send(Err(error));
+                }
             }
-        });
-    }
+        }
+    });
 
     // TODO: create a custom internal error
     let _ = lmdb_writer_sx.send(Ok(TypedChunk::Documents(original_documents_chunk)));
@@ -299,9 +306,13 @@ fn send_and_extract_flattened_documents_data(
     flattened_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
+    searchable_fields: &Option<HashSet<FieldId>>,
+    faceted_fields: &HashSet<FieldId>,
     primary_key_id: FieldId,
     geo_fields_ids: Option<(FieldId, FieldId)>,
-    settings_diff: Arc<InnerIndexSettingsDiff>,
+    stop_words: &Option<fst::Set<Vec<u8>>>,
+    allowed_separators: &Option<&[&str]>,
+    dictionary: &Option<&[&str]>,
     max_positions_per_attributes: Option<u32>,
 ) -> Result<(
     grenad::Reader<CursorClonableMmap>,
@@ -330,7 +341,10 @@ fn send_and_extract_flattened_documents_data(
                     extract_docid_word_positions(
                         flattened_documents_chunk.clone(),
                         indexer,
-                        &settings_diff,
+                        searchable_fields,
+                        stop_words.as_ref(),
+                        *allowed_separators,
+                        *dictionary,
                         max_positions_per_attributes,
                     )?;
 
@@ -353,7 +367,7 @@ fn send_and_extract_flattened_documents_data(
                 } = extract_fid_docid_facet_values(
                     flattened_documents_chunk.clone(),
                     indexer,
-                    &settings_diff,
+                    faceted_fields,
                     geo_fields_ids,
                 )?;
 

milli/src/update/index_documents/mod.rs

@@ -6,9 +6,9 @@ mod typed_chunk;
 
 use std::collections::{HashMap, HashSet};
 use std::io::{Read, Seek};
+use std::iter::FromIterator;
 use std::num::NonZeroU32;
 use std::result::Result as StdResult;
-use std::sync::Arc;
 
 use crossbeam_channel::{Receiver, Sender};
 use grenad::{Merger, MergerBuilder};
@@ -33,7 +33,6 @@ use self::helpers::{grenad_obkv_into_chunks, GrenadParameters};
 pub use self::transform::{Transform, TransformOutput};
 use crate::documents::{obkv_to_object, DocumentsBatchReader};
 use crate::error::{Error, InternalError, UserError};
-use crate::thread_pool_no_abort::ThreadPoolNoAbortBuilder;
 pub use crate::update::index_documents::helpers::CursorClonableMmap;
 use crate::update::{
     IndexerConfig, UpdateIndexingStep, WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst,
@@ -260,6 +259,21 @@ where
             .expect("Invalid document addition state")
             .output_from_sorter(self.wtxn, &self.progress)?;
 
+        let new_facets = output.compute_real_facets(self.wtxn, self.index)?;
+        self.index.put_faceted_fields(self.wtxn, &new_facets)?;
+
+        // in case new fields were introduced we're going to recreate the searchable fields.
+        if let Some(faceted_fields) = self.index.user_defined_searchable_fields(self.wtxn)? {
+            // we can't keep references on the faceted fields while we update the index thus we need to own it.
+            let faceted_fields: Vec<String> =
+                faceted_fields.into_iter().map(str::to_string).collect();
+            self.index.put_all_searchable_fields_from_fields_ids_map(
+                self.wtxn,
+                &faceted_fields.iter().map(String::as_ref).collect::<Vec<_>>(),
+                &output.fields_ids_map,
+            )?;
+        }
+
         let indexed_documents = output.documents_count as u64;
         let number_of_documents = self.execute_raw(output)?;
 
@@ -282,35 +296,32 @@ where
 
         let TransformOutput {
             primary_key,
-            mut settings_diff,
+            fields_ids_map,
             field_distribution,
             documents_count,
             original_documents,
             flattened_documents,
         } = output;
 
-        // update the internal facet and searchable list,
-        // because they might have changed due to the nested documents flattening.
-        settings_diff.new.recompute_facets(self.wtxn, self.index)?;
-        settings_diff.new.recompute_searchables(self.wtxn, self.index)?;
-
-        let settings_diff = Arc::new(settings_diff);
+        // The fields_ids_map is put back to the store now so the rest of the transaction sees an
+        // up to date field map.
+        self.index.put_fields_ids_map(self.wtxn, &fields_ids_map)?;
 
         let backup_pool;
         let pool = match self.indexer_config.thread_pool {
             Some(ref pool) => pool,
+            #[cfg(not(test))]
             None => {
-                // We initialize a backup pool with the default
+                // We initialize a bakcup pool with the default
                 // settings if none have already been set.
-                #[allow(unused_mut)]
-                let mut pool_builder = ThreadPoolNoAbortBuilder::new();
-
-                #[cfg(test)]
-                {
-                    pool_builder = pool_builder.num_threads(1);
-                }
-
-                backup_pool = pool_builder.build()?;
+                backup_pool = rayon::ThreadPoolBuilder::new().build()?;
+                &backup_pool
+            }
+            #[cfg(test)]
+            None => {
+                // We initialize a bakcup pool with the default
+                // settings if none have already been set.
+                backup_pool = rayon::ThreadPoolBuilder::new().num_threads(1).build()?;
                 &backup_pool
             }
         };
@@ -322,8 +333,13 @@ where
         ) = crossbeam_channel::unbounded();
 
         // get the primary key field id
-        let primary_key_id = settings_diff.new.fields_ids_map.id(&primary_key).unwrap();
+        let primary_key_id = fields_ids_map.id(&primary_key).unwrap();
 
+        // get searchable fields for word databases
+        let searchable_fields =
+            self.index.searchable_fields_ids(self.wtxn)?.map(HashSet::from_iter);
+        // get filterable fields for facet databases
+        let faceted_fields = self.index.faceted_fields_ids(self.wtxn)?;
         // get the fid of the `_geo.lat` and `_geo.lng` fields.
         let mut field_id_map = self.index.fields_ids_map(self.wtxn)?;
 
@@ -346,6 +362,12 @@ where
             None => None,
         };
 
+        let stop_words = self.index.stop_words(self.wtxn)?;
+        let separators = self.index.allowed_separators(self.wtxn)?;
+        let dictionary = self.index.dictionary(self.wtxn)?;
+        let exact_attributes = self.index.exact_attributes_ids(self.wtxn)?;
+        let proximity_precision = self.index.proximity_precision(self.wtxn)?.unwrap_or_default();
+
         let pool_params = GrenadParameters {
             chunk_compression_type: self.indexer_config.chunk_compression_type,
             chunk_compression_level: self.indexer_config.chunk_compression_level,
@@ -378,6 +400,8 @@ where
 
         let max_positions_per_attributes = self.indexer_config.max_positions_per_attributes;
 
+        let cloned_embedder = self.embedders.clone();
+
         let mut final_documents_ids = RoaringBitmap::new();
         let mut databases_seen = 0;
         let mut word_position_docids = None;
@@ -386,6 +410,7 @@ where
         let mut exact_word_docids = None;
         let mut chunk_accumulator = ChunkAccumulator::default();
         let mut dimension = HashMap::new();
+        let stop_words = stop_words.map(|sw| sw.map_data(Vec::from).unwrap());
 
         let current_span = tracing::Span::current();
 
@@ -403,6 +428,10 @@ where
                 let flattened_chunk_iter =
                     grenad_obkv_into_chunks(flattened_documents, pool_params, documents_chunk_size);
 
+                let separators: Option<Vec<_>> =
+                    separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
+                let dictionary: Option<Vec<_>> =
+                    dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
                 let result = original_chunk_iter.and_then(|original_chunk| {
                     let flattened_chunk = flattened_chunk_iter?;
                     // extract all databases from the chunked obkv douments
@@ -411,10 +440,18 @@ where
                         flattened_chunk,
                         pool_params,
                         lmdb_writer_sx.clone(),
+                        searchable_fields,
+                        faceted_fields,
                         primary_key_id,
                         geo_fields_ids,
-                        settings_diff.clone(),
+                        field_id_map,
+                        stop_words,
+                        separators.as_deref(),
+                        dictionary.as_deref(),
                         max_positions_per_attributes,
+                        exact_attributes,
+                        proximity_precision,
+                        cloned_embedder,
                     )
                 });
 
@@ -534,7 +571,7 @@ where
             }
 
             Ok(())
-        }).map_err(InternalError::from)??;
+        })?;
 
         // We write the field distribution into the main database
         self.index.put_field_distribution(self.wtxn, &field_distribution)?;
@@ -563,8 +600,7 @@ where
                     writer.build(wtxn, &mut rng, None)?;
                 }
                 Result::Ok(())
-            })
-            .map_err(InternalError::from)??;
+            })?;
         }
 
         self.execute_prefix_databases(

milli/src/update/index_documents/transform.rs

@@ -1,11 +1,12 @@
 use std::borrow::Cow;
 use std::collections::btree_map::Entry as BEntry;
 use std::collections::hash_map::Entry as HEntry;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fs::File;
 use std::io::{Read, Seek};
 
 use fxhash::FxHashMap;
+use heed::RoTxn;
 use itertools::Itertools;
 use obkv::{KvReader, KvReaderU16, KvWriter};
 use roaring::RoaringBitmap;
@@ -20,17 +21,14 @@ use super::{IndexDocumentsMethod, IndexerConfig};
 use crate::documents::{DocumentsBatchIndex, EnrichedDocument, EnrichedDocumentsBatchReader};
 use crate::error::{Error, InternalError, UserError};
 use crate::index::{db_name, main_key};
-use crate::update::del_add::{
-    del_add_from_two_obkvs, into_del_add_obkv, DelAdd, DelAddOperation, KvReaderDelAdd,
-};
+use crate::update::del_add::{into_del_add_obkv, DelAdd, DelAddOperation, KvReaderDelAdd};
 use crate::update::index_documents::GrenadParameters;
-use crate::update::settings::{InnerIndexSettings, InnerIndexSettingsDiff};
-use crate::update::{AvailableDocumentsIds, UpdateIndexingStep};
+use crate::update::{AvailableDocumentsIds, ClearDocuments, UpdateIndexingStep};
 use crate::{FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldsIdsMap, Index, Result};
 
 pub struct TransformOutput {
     pub primary_key: String,
-    pub settings_diff: InnerIndexSettingsDiff,
+    pub fields_ids_map: FieldsIdsMap,
     pub field_distribution: FieldDistribution,
     pub documents_count: usize,
     pub original_documents: File,
@@ -284,9 +282,7 @@ impl<'a, 'i> Transform<'a, 'i> {
                     self.original_sorter
                         .insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?;
                     let base_obkv = KvReader::new(base_obkv);
-                    if let Some(flattened_obkv) =
-                        Self::flatten_from_fields_ids_map(&base_obkv, &mut self.fields_ids_map)?
-                    {
+                    if let Some(flattened_obkv) = self.flatten_from_fields_ids_map(base_obkv)? {
                         // we recreate our buffer with the flattened documents
                         document_sorter_value_buffer.clear();
                         document_sorter_value_buffer.push(Operation::Addition as u8);
@@ -319,9 +315,7 @@ impl<'a, 'i> Transform<'a, 'i> {
                     .insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?;
 
                 let flattened_obkv = KvReader::new(&obkv_buffer);
-                if let Some(obkv) =
-                    Self::flatten_from_fields_ids_map(&flattened_obkv, &mut self.fields_ids_map)?
-                {
+                if let Some(obkv) = self.flatten_from_fields_ids_map(flattened_obkv)? {
                     document_sorter_value_buffer.clear();
                     document_sorter_value_buffer.push(Operation::Addition as u8);
                     into_del_add_obkv(
@@ -530,9 +524,7 @@ impl<'a, 'i> Transform<'a, 'i> {
 
         // flatten it and push it as to delete in the flattened_sorter
         let flattened_obkv = KvReader::new(base_obkv);
-        if let Some(obkv) =
-            Self::flatten_from_fields_ids_map(&flattened_obkv, &mut self.fields_ids_map)?
-        {
+        if let Some(obkv) = self.flatten_from_fields_ids_map(flattened_obkv)? {
             // we recreate our buffer with the flattened documents
             document_sorter_value_buffer.clear();
             document_sorter_value_buffer.push(Operation::Deletion as u8);
@@ -549,15 +541,8 @@ impl<'a, 'i> Transform<'a, 'i> {
 
     // Flatten a document from the fields ids map contained in self and insert the new
     // created fields. Returns `None` if the document doesn't need to be flattened.
-    #[tracing::instrument(
-        level = "trace",
-        skip(obkv, fields_ids_map),
-        target = "indexing::transform"
-    )]
-    fn flatten_from_fields_ids_map(
-        obkv: &KvReader<FieldId>,
-        fields_ids_map: &mut FieldsIdsMap,
-    ) -> Result<Option<Vec<u8>>> {
+    #[tracing::instrument(level = "trace", skip(self, obkv), target = "indexing::transform")]
+    fn flatten_from_fields_ids_map(&mut self, obkv: KvReader<FieldId>) -> Result<Option<Vec<u8>>> {
         if obkv
             .iter()
             .all(|(_, value)| !json_depth_checker::should_flatten_from_unchecked_slice(value))
@@ -578,7 +563,7 @@ impl<'a, 'i> Transform<'a, 'i> {
         // all the raw values get inserted directly in the `key_value` vec.
         for (key, value) in obkv.iter() {
             if json_depth_checker::should_flatten_from_unchecked_slice(value) {
-                let key = fields_ids_map.name(key).ok_or(FieldIdMapMissingEntry::FieldId {
+                let key = self.fields_ids_map.name(key).ok_or(FieldIdMapMissingEntry::FieldId {
                     field_id: key,
                     process: "Flatten from fields ids map.",
                 })?;
@@ -596,7 +581,7 @@ impl<'a, 'i> Transform<'a, 'i> {
         // Once we have the flattened version we insert all the new generated fields_ids
         // (if any) in the fields ids map and serialize the value.
         for (key, value) in flattened.into_iter() {
-            let fid = fields_ids_map.insert(&key).ok_or(UserError::AttributeLimitReached)?;
+            let fid = self.fields_ids_map.insert(&key).ok_or(UserError::AttributeLimitReached)?;
             let value = serde_json::to_vec(&value).map_err(InternalError::SerdeJson)?;
             key_value.push((fid, value.into()));
         }
@@ -807,19 +792,9 @@ impl<'a, 'i> Transform<'a, 'i> {
             fst_new_external_documents_ids_builder.insert(key, value)
         })?;
 
-        let old_inner_settings = InnerIndexSettings::from_index(self.index, wtxn)?;
-        let mut new_inner_settings = old_inner_settings.clone();
-        new_inner_settings.fields_ids_map = self.fields_ids_map;
-        let settings_diff = InnerIndexSettingsDiff {
-            old: old_inner_settings,
-            new: new_inner_settings,
-            embedding_configs_updated: false,
-            settings_update_only: false,
-        };
-
         Ok(TransformOutput {
             primary_key,
-            settings_diff,
+            fields_ids_map: self.fields_ids_map,
             field_distribution,
             documents_count: self.documents_count,
             original_documents: original_documents.into_inner().map_err(|err| err.into_error())?,
@@ -829,44 +804,6 @@ impl<'a, 'i> Transform<'a, 'i> {
         })
     }
 
-    /// Rebind the field_ids of the provided document to their values
-    /// based on the field_ids_maps difference between the old and the new settings,
-    /// then fill the provided buffers with delta documents using KvWritterDelAdd.
-    fn rebind_existing_document(
-        old_obkv: KvReader<FieldId>,
-        settings_diff: &InnerIndexSettingsDiff,
-        original_obkv_buffer: &mut Vec<u8>,
-        flattened_obkv_buffer: &mut Vec<u8>,
-    ) -> Result<()> {
-        let mut old_fields_ids_map = settings_diff.old.fields_ids_map.clone();
-        let mut new_fields_ids_map = settings_diff.new.fields_ids_map.clone();
-        let mut obkv_writer = KvWriter::<_, FieldId>::memory();
-        // We iterate over the new `FieldsIdsMap` ids in order and construct the new obkv.
-        for (id, name) in new_fields_ids_map.iter() {
-            if let Some(val) = old_fields_ids_map.id(name).and_then(|id| old_obkv.get(id)) {
-                obkv_writer.insert(id, val)?;
-            }
-        }
-        let data = obkv_writer.into_inner()?;
-        let new_obkv = KvReader::<FieldId>::new(&data);
-
-        // take the non-flattened version if flatten_from_fields_ids_map returns None.
-        let old_flattened = Self::flatten_from_fields_ids_map(&old_obkv, &mut old_fields_ids_map)?;
-        let old_flattened =
-            old_flattened.as_deref().map_or_else(|| old_obkv, KvReader::<FieldId>::new);
-        let new_flattened = Self::flatten_from_fields_ids_map(&new_obkv, &mut new_fields_ids_map)?;
-        let new_flattened =
-            new_flattened.as_deref().map_or_else(|| new_obkv, KvReader::<FieldId>::new);
-
-        original_obkv_buffer.clear();
-        flattened_obkv_buffer.clear();
-
-        del_add_from_two_obkvs(&old_obkv, &new_obkv, original_obkv_buffer)?;
-        del_add_from_two_obkvs(&old_flattened, &new_flattened, flattened_obkv_buffer)?;
-
-        Ok(())
-    }
-
     /// Clear all databases. Returns a `TransformOutput` with a file that contains the documents
     /// of the index with the attributes reordered accordingly to the `FieldsIdsMap` given as argument.
     ///
@@ -874,7 +811,8 @@ impl<'a, 'i> Transform<'a, 'i> {
     pub fn prepare_for_documents_reindexing(
         self,
         wtxn: &mut heed::RwTxn<'i>,
-        settings_diff: InnerIndexSettingsDiff,
+        old_fields_ids_map: FieldsIdsMap,
+        mut new_fields_ids_map: FieldsIdsMap,
     ) -> Result<TransformOutput> {
         // There already has been a document addition, the primary key should be set by now.
         let primary_key = self
@@ -910,27 +848,78 @@ impl<'a, 'i> Transform<'a, 'i> {
             self.indexer_settings.max_memory.map(|mem| mem / 2),
         );
 
-        let mut original_obkv_buffer = Vec::new();
-        let mut flattened_obkv_buffer = Vec::new();
+        let mut obkv_buffer = Vec::new();
         let mut document_sorter_key_buffer = Vec::new();
+        let mut document_sorter_value_buffer = Vec::new();
         for result in self.index.external_documents_ids().iter(wtxn)? {
             let (external_id, docid) = result?;
-            let old_obkv = self.index.documents.get(wtxn, &docid)?.ok_or(
+            let obkv = self.index.documents.get(wtxn, &docid)?.ok_or(
                 InternalError::DatabaseMissingEntry { db_name: db_name::DOCUMENTS, key: None },
             )?;
 
-            Self::rebind_existing_document(
-                old_obkv,
-                &settings_diff,
-                &mut original_obkv_buffer,
-                &mut flattened_obkv_buffer,
-            )?;
+            obkv_buffer.clear();
+            let mut obkv_writer = KvWriter::<_, FieldId>::new(&mut obkv_buffer);
+
+            // We iterate over the new `FieldsIdsMap` ids in order and construct the new obkv.
+            for (id, name) in new_fields_ids_map.iter() {
+                if let Some(val) = old_fields_ids_map.id(name).and_then(|id| obkv.get(id)) {
+                    obkv_writer.insert(id, val)?;
+                }
+            }
+
+            let buffer = obkv_writer.into_inner()?;
 
             document_sorter_key_buffer.clear();
             document_sorter_key_buffer.extend_from_slice(&docid.to_be_bytes());
             document_sorter_key_buffer.extend_from_slice(external_id.as_bytes());
-            original_sorter.insert(&document_sorter_key_buffer, &original_obkv_buffer)?;
-            flattened_sorter.insert(docid.to_be_bytes(), &flattened_obkv_buffer)?;
+            document_sorter_value_buffer.clear();
+            into_del_add_obkv(
+                KvReaderU16::new(buffer),
+                DelAddOperation::Addition,
+                &mut document_sorter_value_buffer,
+            )?;
+            original_sorter.insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?;
+
+            // Once we have the document. We're going to flatten it
+            // and insert it in the flattened sorter.
+            let mut doc = serde_json::Map::new();
+
+            let reader = obkv::KvReader::new(buffer);
+            for (k, v) in reader.iter() {
+                let key = new_fields_ids_map.name(k).ok_or(FieldIdMapMissingEntry::FieldId {
+                    field_id: k,
+                    process: "Accessing field distribution in transform.",
+                })?;
+                let value = serde_json::from_slice::<serde_json::Value>(v)
+                    .map_err(InternalError::SerdeJson)?;
+                doc.insert(key.to_string(), value);
+            }
+
+            let flattened = flatten_serde_json::flatten(&doc);
+
+            // Once we have the flattened version we can convert it back to obkv and
+            // insert all the new generated fields_ids (if any) in the fields ids map.
+            let mut buffer: Vec<u8> = Vec::new();
+            let mut writer = KvWriter::new(&mut buffer);
+            let mut flattened: Vec<_> = flattened.into_iter().collect();
+            // we reorder the field to get all the known field first
+            flattened.sort_unstable_by_key(|(key, _)| {
+                new_fields_ids_map.id(key).unwrap_or(FieldId::MAX)
+            });
+
+            for (key, value) in flattened {
+                let fid =
+                    new_fields_ids_map.insert(&key).ok_or(UserError::AttributeLimitReached)?;
+                let value = serde_json::to_vec(&value).map_err(InternalError::SerdeJson)?;
+                writer.insert(fid, &value)?;
+            }
+            document_sorter_value_buffer.clear();
+            into_del_add_obkv(
+                KvReaderU16::new(&buffer),
+                DelAddOperation::Addition,
+                &mut document_sorter_value_buffer,
+            )?;
+            flattened_sorter.insert(docid.to_be_bytes(), &document_sorter_value_buffer)?;
         }
 
         let grenad_params = GrenadParameters {
@@ -945,14 +934,22 @@ impl<'a, 'i> Transform<'a, 'i> {
 
         let flattened_documents = sorter_into_reader(flattened_sorter, grenad_params)?;
 
-        Ok(TransformOutput {
+        let output = TransformOutput {
             primary_key,
+            fields_ids_map: new_fields_ids_map,
             field_distribution,
-            settings_diff,
             documents_count,
             original_documents: original_documents.into_inner().into_inner(),
             flattened_documents: flattened_documents.into_inner().into_inner(),
-        })
+        };
+
+        let new_facets = output.compute_real_facets(wtxn, self.index)?;
+        self.index.put_faceted_fields(wtxn, &new_facets)?;
+
+        // We clear the full database (words-fst, documents ids and documents content).
+        ClearDocuments::new(wtxn, self.index).execute()?;
+
+        Ok(output)
     }
 }
 
@@ -967,6 +964,20 @@ fn drop_and_reuse<U, T>(mut vec: Vec<U>) -> Vec<T> {
     vec.into_iter().map(|_| unreachable!()).collect()
 }
 
+impl TransformOutput {
+    // find and insert the new field ids
+    pub fn compute_real_facets(&self, rtxn: &RoTxn, index: &Index) -> Result<HashSet<String>> {
+        let user_defined_facets = index.user_defined_faceted_fields(rtxn)?;
+
+        Ok(self
+            .fields_ids_map
+            .names()
+            .filter(|&field| crate::is_faceted(field, &user_defined_facets))
+            .map(|field| field.to_string())
+            .collect())
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

milli/src/update/indexer_config.rs

@@ -1,6 +1,5 @@
 use grenad::CompressionType;
-
-use crate::thread_pool_no_abort::ThreadPoolNoAbort;
+use rayon::ThreadPool;
 
 #[derive(Debug)]
 pub struct IndexerConfig {
@@ -10,7 +9,7 @@ pub struct IndexerConfig {
     pub max_memory: Option<usize>,
     pub chunk_compression_type: CompressionType,
     pub chunk_compression_level: Option<u32>,
-    pub thread_pool: Option<ThreadPoolNoAbort>,
+    pub thread_pool: Option<ThreadPool>,
     pub max_positions_per_attributes: Option<u32>,
     pub skip_index_budget: bool,
 }

milli/src/update/settings.rs

@@ -20,7 +20,7 @@ use crate::update::index_documents::IndexDocumentsMethod;
 use crate::update::{IndexDocuments, UpdateIndexingStep};
 use crate::vector::settings::{check_set, check_unset, EmbedderSource, EmbeddingSettings};
 use crate::vector::{Embedder, EmbeddingConfig, EmbeddingConfigs};
-use crate::{FieldId, FieldsIdsMap, Index, Result};
+use crate::{FieldsIdsMap, Index, Result};
 
 #[derive(Debug, Clone, PartialEq, Eq, Copy)]
 pub enum Setting<T> {
@@ -385,14 +385,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
 
     #[tracing::instrument(
         level = "trace"
-        skip(self, progress_callback, should_abort, settings_diff),
+        skip(self, progress_callback, should_abort, old_fields_ids_map),
         target = "indexing::documents"
     )]
     fn reindex<FP, FA>(
         &mut self,
         progress_callback: &FP,
         should_abort: &FA,
-        settings_diff: InnerIndexSettingsDiff,
+        old_fields_ids_map: FieldsIdsMap,
     ) -> Result<()>
     where
         FP: Fn(UpdateIndexingStep) + Sync,
@@ -400,6 +400,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
     {
         puffin::profile_function!();
 
+        let fields_ids_map = self.index.fields_ids_map(self.wtxn)?;
         // if the settings are set before any document update, we don't need to do anything, and
         // will set the primary key during the first document addition.
         if self.index.number_of_documents(self.wtxn)? == 0 {
@@ -415,7 +416,14 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         )?;
 
         // We clear the databases and remap the documents fields based on the new `FieldsIdsMap`.
-        let output = transform.prepare_for_documents_reindexing(self.wtxn, settings_diff)?;
+        let output = transform.prepare_for_documents_reindexing(
+            self.wtxn,
+            old_fields_ids_map,
+            fields_ids_map,
+        )?;
+
+        let embedder_configs = self.index.embedding_configs(self.wtxn)?;
+        let embedders = self.embedders(embedder_configs)?;
 
         // We index the generated `TransformOutput` which must contain
         // all the documents with fields in the newly defined searchable order.
@@ -428,11 +436,32 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
             &should_abort,
         )?;
 
+        let indexing_builder = indexing_builder.with_embedders(embedders);
         indexing_builder.execute_raw(output)?;
 
         Ok(())
     }
 
+    fn embedders(
+        &self,
+        embedding_configs: Vec<(String, EmbeddingConfig)>,
+    ) -> Result<EmbeddingConfigs> {
+        let res: Result<_> = embedding_configs
+            .into_iter()
+            .map(|(name, EmbeddingConfig { embedder_options, prompt })| {
+                let prompt = Arc::new(prompt.try_into().map_err(crate::Error::from)?);
+
+                let embedder = Arc::new(
+                    Embedder::new(embedder_options.clone())
+                        .map_err(crate::vector::Error::from)
+                        .map_err(crate::Error::from)?,
+                );
+                Ok((name, (embedder, prompt)))
+            })
+            .collect();
+        res.map(EmbeddingConfigs::new)
+    }
+
     fn update_displayed(&mut self) -> Result<bool> {
         match self.displayed_fields {
             Setting::Set(ref fields) => {
@@ -1009,13 +1038,6 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
             }
             Setting::NotSet => false,
         };
-
-        // if any changes force a reindexing
-        // clear the vector database.
-        if update {
-            self.index.vector_arroy.clear(self.wtxn)?;
-        }
-
         Ok(update)
     }
 
@@ -1044,10 +1066,20 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
     {
         self.index.set_updated_at(self.wtxn, &OffsetDateTime::now_utc())?;
 
-        let old_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn)?;
+        // Note: this MUST be before `update_sortable` so that we can get the old value to compare with the updated value afterwards
+
+        let existing_fields: HashSet<_> = self
+            .index
+            .field_distribution(self.wtxn)?
+            .into_iter()
+            .filter_map(|(field, count)| (count != 0).then_some(field))
+            .collect();
+        let old_faceted_fields = self.index.user_defined_faceted_fields(self.wtxn)?;
+        let old_fields_ids_map = self.index.fields_ids_map(self.wtxn)?;
 
-        // never trigger re-indexing
         self.update_displayed()?;
+        self.update_filterable()?;
+        self.update_sortable()?;
         self.update_distinct_field()?;
         self.update_criteria()?;
         self.update_primary_key()?;
@@ -1057,19 +1089,16 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         self.update_max_values_per_facet()?;
         self.update_sort_facet_values_by()?;
         self.update_pagination_max_total_hits()?;
-        self.update_search_cutoff()?;
 
-        // could trigger re-indexing
-        self.update_filterable()?;
-        self.update_sortable()?;
-        self.update_stop_words()?;
-        self.update_non_separator_tokens()?;
-        self.update_separator_tokens()?;
-        self.update_dictionary()?;
-        self.update_synonyms()?;
-        self.update_searchable()?;
-        self.update_exact_attributes()?;
-        self.update_proximity_precision()?;
+        let faceted_updated = self.update_faceted(existing_fields, old_faceted_fields)?;
+        let stop_words_updated = self.update_stop_words()?;
+        let non_separator_tokens_updated = self.update_non_separator_tokens()?;
+        let separator_tokens_updated = self.update_separator_tokens()?;
+        let dictionary_updated = self.update_dictionary()?;
+        let synonyms_updated = self.update_synonyms()?;
+        let searchable_updated = self.update_searchable()?;
+        let exact_attributes_updated = self.update_exact_attributes()?;
+        let proximity_precision = self.update_proximity_precision()?;
         // TODO: very rough approximation of the needs for reindexing where any change will result in
         // a full reindexing.
         // What can be done instead:
@@ -1078,193 +1107,53 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
         // 3. Keep the old vectors but reattempt indexing on a prompt change: only actually changed prompt will need embedding + storage
         let embedding_configs_updated = self.update_embedding_configs()?;
 
-        let new_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn)?;
-        let inner_settings_diff = InnerIndexSettingsDiff {
-            old: old_inner_settings,
-            new: new_inner_settings,
-            embedding_configs_updated,
-            settings_update_only: true,
-        };
+        // never trigger re-indexing
+        self.update_search_cutoff()?;
 
-        if inner_settings_diff.any_reindexing_needed() {
-            self.reindex(&progress_callback, &should_abort, inner_settings_diff)?;
+        if stop_words_updated
+            || non_separator_tokens_updated
+            || separator_tokens_updated
+            || dictionary_updated
+            || faceted_updated
+            || synonyms_updated
+            || searchable_updated
+            || exact_attributes_updated
+            || proximity_precision
+            || embedding_configs_updated
+        {
+            self.reindex(&progress_callback, &should_abort, old_fields_ids_map)?;
         }
 
         Ok(())
     }
-}
 
-pub struct InnerIndexSettingsDiff {
-    pub(crate) old: InnerIndexSettings,
-    pub(crate) new: InnerIndexSettings,
-
-    // TODO: compare directly the embedders.
-    pub(crate) embedding_configs_updated: bool,
-
-    pub(crate) settings_update_only: bool,
-}
-
-impl InnerIndexSettingsDiff {
-    pub fn any_reindexing_needed(&self) -> bool {
-        self.reindex_searchable() || self.reindex_facets() || self.reindex_vectors()
-    }
-
-    pub fn reindex_searchable(&self) -> bool {
-        self.old
-            .fields_ids_map
-            .iter()
-            .zip(self.new.fields_ids_map.iter())
-            .any(|(old, new)| old != new)
-            || self.old.stop_words.as_ref().map(|set| set.as_fst().as_bytes())
-                != self.new.stop_words.as_ref().map(|set| set.as_fst().as_bytes())
-            || self.old.allowed_separators != self.new.allowed_separators
-            || self.old.dictionary != self.new.dictionary
-            || self.old.user_defined_searchable_fields != self.new.user_defined_searchable_fields
-            || self.old.exact_attributes != self.new.exact_attributes
-            || self.old.proximity_precision != self.new.proximity_precision
-    }
-
-    pub fn reindex_facets(&self) -> bool {
-        let existing_fields = &self.new.existing_fields;
+    fn update_faceted(
+        &self,
+        existing_fields: HashSet<String>,
+        old_faceted_fields: HashSet<String>,
+    ) -> Result<bool> {
         if existing_fields.iter().any(|field| field.contains('.')) {
-            return true;
+            return Ok(true);
         }
 
-        let old_faceted_fields = &self.old.user_defined_faceted_fields;
         if old_faceted_fields.iter().any(|field| field.contains('.')) {
-            return true;
+            return Ok(true);
         }
 
         // If there is new faceted fields we indicate that we must reindex as we must
         // index new fields as facets. It means that the distinct attribute,
         // an Asc/Desc criterion or a filtered attribute as be added or removed.
-        let new_faceted_fields = &self.new.user_defined_faceted_fields;
+        let new_faceted_fields = self.index.user_defined_faceted_fields(self.wtxn)?;
+
         if new_faceted_fields.iter().any(|field| field.contains('.')) {
-            return true;
+            return Ok(true);
         }
 
         let faceted_updated =
-            (existing_fields - old_faceted_fields) != (existing_fields - new_faceted_fields);
+            (&existing_fields - &old_faceted_fields) != (&existing_fields - &new_faceted_fields);
 
-        self.old
-            .fields_ids_map
-            .iter()
-            .zip(self.new.fields_ids_map.iter())
-            .any(|(old, new)| old != new)
-            || faceted_updated
+        Ok(faceted_updated)
     }
-
-    pub fn reindex_vectors(&self) -> bool {
-        self.embedding_configs_updated
-    }
-
-    pub fn settings_update_only(&self) -> bool {
-        self.settings_update_only
-    }
-}
-
-#[derive(Clone)]
-pub(crate) struct InnerIndexSettings {
-    pub stop_words: Option<fst::Set<Vec<u8>>>,
-    pub allowed_separators: Option<BTreeSet<String>>,
-    pub dictionary: Option<BTreeSet<String>>,
-    pub fields_ids_map: FieldsIdsMap,
-    pub user_defined_faceted_fields: HashSet<String>,
-    pub user_defined_searchable_fields: Option<Vec<String>>,
-    pub faceted_fields_ids: HashSet<FieldId>,
-    pub searchable_fields_ids: Option<Vec<FieldId>>,
-    pub exact_attributes: HashSet<FieldId>,
-    pub proximity_precision: ProximityPrecision,
-    pub embedding_configs: EmbeddingConfigs,
-    pub existing_fields: HashSet<String>,
-}
-
-impl InnerIndexSettings {
-    pub fn from_index(index: &Index, rtxn: &heed::RoTxn) -> Result<Self> {
-        let stop_words = index.stop_words(rtxn)?;
-        let stop_words = stop_words.map(|sw| sw.map_data(Vec::from).unwrap());
-        let allowed_separators = index.allowed_separators(rtxn)?;
-        let dictionary = index.dictionary(rtxn)?;
-        let fields_ids_map = index.fields_ids_map(rtxn)?;
-        let user_defined_searchable_fields = index.user_defined_searchable_fields(rtxn)?;
-        let user_defined_searchable_fields =
-            user_defined_searchable_fields.map(|sf| sf.into_iter().map(String::from).collect());
-        let user_defined_faceted_fields = index.user_defined_faceted_fields(rtxn)?;
-        let searchable_fields_ids = index.searchable_fields_ids(rtxn)?;
-        let faceted_fields_ids = index.faceted_fields_ids(rtxn)?;
-        let exact_attributes = index.exact_attributes_ids(rtxn)?;
-        let proximity_precision = index.proximity_precision(rtxn)?.unwrap_or_default();
-        let embedding_configs = embedders(index.embedding_configs(rtxn)?)?;
-        let existing_fields: HashSet<_> = index
-            .field_distribution(rtxn)?
-            .into_iter()
-            .filter_map(|(field, count)| (count != 0).then_some(field))
-            .collect();
-
-        Ok(Self {
-            stop_words,
-            allowed_separators,
-            dictionary,
-            fields_ids_map,
-            user_defined_faceted_fields,
-            user_defined_searchable_fields,
-            faceted_fields_ids,
-            searchable_fields_ids,
-            exact_attributes,
-            proximity_precision,
-            embedding_configs,
-            existing_fields,
-        })
-    }
-
-    // find and insert the new field ids
-    pub fn recompute_facets(&mut self, wtxn: &mut heed::RwTxn, index: &Index) -> Result<()> {
-        let new_facets = self
-            .fields_ids_map
-            .names()
-            .filter(|&field| crate::is_faceted(field, &self.user_defined_faceted_fields))
-            .map(|field| field.to_string())
-            .collect();
-        index.put_faceted_fields(wtxn, &new_facets)?;
-
-        self.faceted_fields_ids = index.faceted_fields_ids(wtxn)?;
-        Ok(())
-    }
-
-    // find and insert the new field ids
-    pub fn recompute_searchables(&mut self, wtxn: &mut heed::RwTxn, index: &Index) -> Result<()> {
-        // in case new fields were introduced we're going to recreate the searchable fields.
-        if let Some(searchable_fields) = self.user_defined_searchable_fields.as_ref() {
-            let searchable_fields =
-                searchable_fields.iter().map(String::as_ref).collect::<Vec<_>>();
-            index.put_all_searchable_fields_from_fields_ids_map(
-                wtxn,
-                &searchable_fields,
-                &self.fields_ids_map,
-            )?;
-            let searchable_fields_ids = index.searchable_fields_ids(wtxn)?;
-            self.searchable_fields_ids = searchable_fields_ids;
-        }
-
-        Ok(())
-    }
-}
-
-fn embedders(embedding_configs: Vec<(String, EmbeddingConfig)>) -> Result<EmbeddingConfigs> {
-    let res: Result<_> = embedding_configs
-        .into_iter()
-        .map(|(name, EmbeddingConfig { embedder_options, prompt })| {
-            let prompt = Arc::new(prompt.try_into().map_err(crate::Error::from)?);
-
-            let embedder = Arc::new(
-                Embedder::new(embedder_options.clone())
-                    .map_err(crate::vector::Error::from)
-                    .map_err(crate::Error::from)?,
-            );
-            Ok((name, (embedder, prompt)))
-        })
-        .collect();
-    res.map(EmbeddingConfigs::new)
 }
 
 fn validate_prompt(
@@ -1754,70 +1643,6 @@ mod tests {
             .unwrap()
             .count();
         assert_eq!(count, 4);
-
-        // Set the filterable fields to be the age and the name.
-        index
-            .update_settings(|settings| {
-                settings.set_filterable_fields(hashset! { S("age"),  S("name") });
-            })
-            .unwrap();
-
-        // Check that the displayed fields are correctly set.
-        let rtxn = index.read_txn().unwrap();
-        let fields_ids = index.filterable_fields(&rtxn).unwrap();
-        assert_eq!(fields_ids, hashset! { S("age"),  S("name") });
-
-        let rtxn = index.read_txn().unwrap();
-        // Only count the field_id 0 and level 0 facet values.
-        let count = index
-            .facet_id_f64_docids
-            .remap_key_type::<Bytes>()
-            .prefix_iter(&rtxn, &[0, 1, 0])
-            .unwrap()
-            .count();
-        assert_eq!(count, 4);
-
-        let rtxn = index.read_txn().unwrap();
-        // Only count the field_id 0 and level 0 facet values.
-        let count = index
-            .facet_id_string_docids
-            .remap_key_type::<Bytes>()
-            .prefix_iter(&rtxn, &[0, 0])
-            .unwrap()
-            .count();
-        assert_eq!(count, 5);
-
-        // Remove the age from the filterable fields.
-        index
-            .update_settings(|settings| {
-                settings.set_filterable_fields(hashset! { S("name") });
-            })
-            .unwrap();
-
-        // Check that the displayed fields are correctly set.
-        let rtxn = index.read_txn().unwrap();
-        let fields_ids = index.filterable_fields(&rtxn).unwrap();
-        assert_eq!(fields_ids, hashset! { S("name") });
-
-        let rtxn = index.read_txn().unwrap();
-        // Only count the field_id 0 and level 0 facet values.
-        let count = index
-            .facet_id_f64_docids
-            .remap_key_type::<Bytes>()
-            .prefix_iter(&rtxn, &[0, 1, 0])
-            .unwrap()
-            .count();
-        assert_eq!(count, 0);
-
-        let rtxn = index.read_txn().unwrap();
-        // Only count the field_id 0 and level 0 facet values.
-        let count = index
-            .facet_id_string_docids
-            .remap_key_type::<Bytes>()
-            .prefix_iter(&rtxn, &[0, 0])
-            .unwrap()
-            .count();
-        assert_eq!(count, 5);
     }
 
     #[test]

milli/src/vector/error.rs

@@ -3,7 +3,6 @@ use std::path::PathBuf;
 use hf_hub::api::sync::ApiError;
 
 use crate::error::FaultSource;
-use crate::PanicCatched;
 
 #[derive(Debug, thiserror::Error)]
 #[error("Error while generating embeddings: {inner}")]
@@ -81,8 +80,6 @@ pub enum EmbedErrorKind {
     OpenAiUnexpectedDimension(usize, usize),
     #[error("no embedding was produced")]
     MissingEmbedding,
-    #[error(transparent)]
-    PanicInThreadPool(#[from] PanicCatched),
 }
 
 impl EmbedError {

milli/src/vector/mod.rs

@@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize};
 
 use self::error::{EmbedError, NewEmbedderError};
 use crate::prompt::{Prompt, PromptData};
-use crate::ThreadPoolNoAbort;
 
 pub mod error;
 pub mod hf;
@@ -255,7 +254,7 @@ impl Embedder {
     pub fn embed_chunks(
         &self,
         text_chunks: Vec<Vec<String>>,
-        threads: &ThreadPoolNoAbort,
+        threads: &rayon::ThreadPool,
     ) -> std::result::Result<Vec<Vec<Embeddings<f32>>>, EmbedError> {
         match self {
             Embedder::HuggingFace(embedder) => embedder.embed_chunks(text_chunks),

milli/src/vector/ollama.rs

@@ -3,8 +3,6 @@ use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
 use super::error::{EmbedError, EmbedErrorKind, NewEmbedderError, NewEmbedderErrorKind};
 use super::rest::{Embedder as RestEmbedder, EmbedderOptions as RestEmbedderOptions};
 use super::{DistributionShift, Embeddings};
-use crate::error::FaultSource;
-use crate::ThreadPoolNoAbort;
 
 #[derive(Debug)]
 pub struct Embedder {
@@ -73,16 +71,11 @@ impl Embedder {
     pub fn embed_chunks(
         &self,
         text_chunks: Vec<Vec<String>>,
-        threads: &ThreadPoolNoAbort,
+        threads: &rayon::ThreadPool,
     ) -> Result<Vec<Vec<Embeddings<f32>>>, EmbedError> {
-        threads
-            .install(move || {
-                text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
-            })
-            .map_err(|error| EmbedError {
-                kind: EmbedErrorKind::PanicInThreadPool(error),
-                fault: FaultSource::Bug,
-            })?
+        threads.install(move || {
+            text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
+        })
     }
 
     pub fn chunk_count_hint(&self) -> usize {

milli/src/vector/openai.rs

@@ -4,9 +4,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator as _};
 use super::error::{EmbedError, NewEmbedderError};
 use super::rest::{Embedder as RestEmbedder, EmbedderOptions as RestEmbedderOptions};
 use super::{DistributionShift, Embeddings};
-use crate::error::FaultSource;
 use crate::vector::error::EmbedErrorKind;
-use crate::ThreadPoolNoAbort;
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
@@ -243,16 +241,11 @@ impl Embedder {
     pub fn embed_chunks(
         &self,
         text_chunks: Vec<Vec<String>>,
-        threads: &ThreadPoolNoAbort,
+        threads: &rayon::ThreadPool,
     ) -> Result<Vec<Vec<Embeddings<f32>>>, EmbedError> {
-        threads
-            .install(move || {
-                text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
-            })
-            .map_err(|error| EmbedError {
-                kind: EmbedErrorKind::PanicInThreadPool(error),
-                fault: FaultSource::Bug,
-            })?
+        threads.install(move || {
+            text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
+        })
     }
 
     pub fn chunk_count_hint(&self) -> usize {

milli/src/vector/rest.rs

@@ -2,12 +2,9 @@ use deserr::Deserr;
 use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
 use serde::{Deserialize, Serialize};
 
-use super::error::EmbedErrorKind;
 use super::{
     DistributionShift, EmbedError, Embedding, Embeddings, NewEmbedderError, REQUEST_PARALLELISM,
 };
-use crate::error::FaultSource;
-use crate::ThreadPoolNoAbort;
 
 // retrying in case of failure
 
@@ -161,16 +158,11 @@ impl Embedder {
     pub fn embed_chunks(
         &self,
         text_chunks: Vec<Vec<String>>,
-        threads: &ThreadPoolNoAbort,
+        threads: &rayon::ThreadPool,
     ) -> Result<Vec<Vec<Embeddings<f32>>>, EmbedError> {
-        threads
-            .install(move || {
-                text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
-            })
-            .map_err(|error| EmbedError {
-                kind: EmbedErrorKind::PanicInThreadPool(error),
-                fault: FaultSource::Bug,
-            })?
+        threads.install(move || {
+            text_chunks.into_par_iter().map(move |chunk| self.embed(chunk)).collect()
+        })
     }
 
     pub fn chunk_count_hint(&self) -> usize {

milli/src/vector/settings.rs

@@ -301,14 +301,10 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
     fn from(value: EmbeddingConfig) -> Self {
         let EmbeddingConfig { embedder_options, prompt } = value;
         match embedder_options {
-            super::EmbedderOptions::HuggingFace(super::hf::EmbedderOptions {
-                model,
-                revision,
-                distribution,
-            }) => Self {
+            super::EmbedderOptions::HuggingFace(options) => Self {
                 source: Setting::Set(EmbedderSource::HuggingFace),
-                model: Setting::Set(model),
-                revision: revision.map(Setting::Set).unwrap_or_default(),
+                model: Setting::Set(options.model),
+                revision: options.revision.map(Setting::Set).unwrap_or_default(),
                 api_key: Setting::NotSet,
                 dimensions: Setting::NotSet,
                 document_template: Setting::Set(prompt.template),
@@ -318,19 +314,14 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 path_to_embeddings: Setting::NotSet,
                 embedding_object: Setting::NotSet,
                 input_type: Setting::NotSet,
-                distribution: distribution.map(Setting::Set).unwrap_or_default(),
+                distribution: options.distribution.map(Setting::Set).unwrap_or_default(),
             },
-            super::EmbedderOptions::OpenAi(super::openai::EmbedderOptions {
-                api_key,
-                embedding_model,
-                dimensions,
-                distribution,
-            }) => Self {
+            super::EmbedderOptions::OpenAi(options) => Self {
                 source: Setting::Set(EmbedderSource::OpenAi),
-                model: Setting::Set(embedding_model.name().to_owned()),
+                model: Setting::Set(options.embedding_model.name().to_owned()),
                 revision: Setting::NotSet,
-                api_key: api_key.map(Setting::Set).unwrap_or_default(),
-                dimensions: dimensions.map(Setting::Set).unwrap_or_default(),
+                api_key: options.api_key.map(Setting::Set).unwrap_or_default(),
+                dimensions: options.dimensions.map(Setting::Set).unwrap_or_default(),
                 document_template: Setting::Set(prompt.template),
                 url: Setting::NotSet,
                 query: Setting::NotSet,
@@ -338,37 +329,29 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 path_to_embeddings: Setting::NotSet,
                 embedding_object: Setting::NotSet,
                 input_type: Setting::NotSet,
-                distribution: distribution.map(Setting::Set).unwrap_or_default(),
+                distribution: options.distribution.map(Setting::Set).unwrap_or_default(),
             },
-            super::EmbedderOptions::Ollama(super::ollama::EmbedderOptions {
-                embedding_model,
-                url,
-                api_key,
-                distribution,
-            }) => Self {
+            super::EmbedderOptions::Ollama(options) => Self {
                 source: Setting::Set(EmbedderSource::Ollama),
-                model: Setting::Set(embedding_model),
+                model: Setting::Set(options.embedding_model.to_owned()),
                 revision: Setting::NotSet,
-                api_key: api_key.map(Setting::Set).unwrap_or_default(),
+                api_key: Setting::NotSet,
                 dimensions: Setting::NotSet,
                 document_template: Setting::Set(prompt.template),
-                url: url.map(Setting::Set).unwrap_or_default(),
+                url: Setting::NotSet,
                 query: Setting::NotSet,
                 input_field: Setting::NotSet,
                 path_to_embeddings: Setting::NotSet,
                 embedding_object: Setting::NotSet,
                 input_type: Setting::NotSet,
-                distribution: distribution.map(Setting::Set).unwrap_or_default(),
+                distribution: options.distribution.map(Setting::Set).unwrap_or_default(),
             },
-            super::EmbedderOptions::UserProvided(super::manual::EmbedderOptions {
-                dimensions,
-                distribution,
-            }) => Self {
+            super::EmbedderOptions::UserProvided(options) => Self {
                 source: Setting::Set(EmbedderSource::UserProvided),
                 model: Setting::NotSet,
                 revision: Setting::NotSet,
                 api_key: Setting::NotSet,
-                dimensions: Setting::Set(dimensions),
+                dimensions: Setting::Set(options.dimensions),
                 document_template: Setting::NotSet,
                 url: Setting::NotSet,
                 query: Setting::NotSet,
@@ -376,7 +359,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
                 path_to_embeddings: Setting::NotSet,
                 embedding_object: Setting::NotSet,
                 input_type: Setting::NotSet,
-                distribution: distribution.map(Setting::Set).unwrap_or_default(),
+                distribution: options.distribution.map(Setting::Set).unwrap_or_default(),
             },
             super::EmbedderOptions::Rest(super::rest::EmbedderOptions {
                 api_key,

tracing-trace/src/processor/firefox_profiler.rs

@@ -217,7 +217,9 @@ fn add_memory_samples(
     memory_counters: &mut Option<MemoryCounterHandles>,
     last_memory: &mut MemoryStats,
 ) -> Option<MemoryStats> {
-    let stats = memory?;
+    let Some(stats) = memory else {
+        return None;
+    };
 
     let memory_counters =
         memory_counters.get_or_insert_with(|| MemoryCounterHandles::new(profile, main));

workloads/movies-subset-hf-embeddings.json

@@ -1,68 +0,0 @@
-{
-  "name": "movies-subset-hf-embeddings",
-  "run_count": 5,
-  "extra_cli_args": [
-    "--max-indexing-threads=4"
-  ],
-  "assets": {
-    "movies-100.json": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies-100.json",
-      "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
-    }
-  },
-  "commands": [
-    {
-      "route": "experimental-features",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "vectorStore": true
-        }
-      },
-      "synchronous": "DontWait"
-    },
-    {
-      "route": "indexes/movies/settings",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "searchableAttributes": [
-            "title",
-            "overview"
-          ],
-          "filterableAttributes": [
-            "genres",
-            "release_date"
-          ],
-          "sortableAttributes": [
-            "release_date"
-          ]
-        }
-      },
-      "synchronous": "WaitForTask"
-    },
-    {
-      "route": "indexes/movies/settings",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "embedders": {
-            "default": {
-              "source": "huggingFace"
-            }
-          }
-        }
-      },
-      "synchronous": "WaitForTask"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "movies-100.json"
-      },
-      "synchronous": "WaitForTask"
-    }
-  ]
-}

workloads/settings-add-embeddings.json

@@ -1,72 +0,0 @@
-{
-  "name": "settings-add-embeddings-hf",
-  "run_count": 5,
-  "extra_cli_args": [
-    "--max-indexing-threads=4"
-  ],
-  "assets": {
-    "movies-100.json": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies-100.json",
-      "sha256": "d215e395e4240f12f03b8f1f68901eac82d9e7ded5b462cbf4a6b8efde76c6c6"
-    }
-  },
-  "commands": [
-    {
-      "route": "experimental-features",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "vectorStore": true
-        }
-      },
-      "synchronous": "DontWait"
-    },
-    {
-      "route": "indexes/movies/settings",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "searchableAttributes": [
-            "title",
-            "overview"
-          ],
-          "filterableAttributes": [
-            "genres",
-            "release_date"
-          ],
-          "sortableAttributes": [
-            "release_date"
-          ]
-        }
-      },
-      "synchronous": "DontWait"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "movies-100.json"
-      },
-      "synchronous": "WaitForTask"
-    },
-    {
-      "route": "indexes/movies/settings",
-      "method": "PATCH",
-      "body": {
-        "inline": {
-          "embedders": {
-            "default": {
-              "source": "huggingFace",
-              "model": null,
-              "revision": null,
-              "documentTemplate": null,
-              "distribution": null
-            }
-          }
-        }
-      },
-      "synchronous": "WaitForTask"
-    }
-  ]
-}

workloads/settings-add-remove-filters.json

@@ -1,6 +1,6 @@
 {
   "name": "settings-add-remove-filters.json",
-  "run_count": 5,
+  "run_count": 2,
   "extra_cli_args": [
     "--max-indexing-threads=4"
   ],

workloads/settings-proximity-precision.json

@@ -1,6 +1,6 @@
 {
   "name": "settings-proximity-precision.json",
-  "run_count": 5,
+  "run_count": 2,
   "extra_cli_args": [
     "--max-indexing-threads=4"
   ],

workloads/settings-remove-add-swap-searchable.json

@@ -1,6 +1,6 @@
 {
   "name": "settings-remove-add-swap-searchable.json",
-  "run_count": 5,
+  "run_count": 2,
   "extra_cli_args": [
     "--max-indexing-threads=4"
   ],

workloads/settings-typo.json

@@ -1,6 +1,6 @@
 {
   "name": "settings-typo.json",
-  "run_count": 5,
+  "run_count": 2,
   "extra_cli_args": [
     "--max-indexing-threads=4"
   ],