From 6c1739218c26bfe269fa23ddd23b3623db0d5483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 May 2025 18:34:11 +0200 Subject: [PATCH] Settings in queue --- crates/index-scheduler/src/lib.rs | 17 +- crates/index-scheduler/src/settings/chat.rs | 432 ++++++++++++++++++ crates/index-scheduler/src/settings/mod.rs | 3 + crates/meilisearch-types/src/settings.rs | 1 + .../meilisearch/src/routes/settings/chat.rs | 2 +- 5 files changed, 443 insertions(+), 12 deletions(-) create mode 100644 crates/index-scheduler/src/settings/chat.rs create mode 100644 crates/index-scheduler/src/settings/mod.rs diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index d0b924a91..339b18086 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -28,6 +28,7 @@ mod lru; mod processing; mod queue; mod scheduler; +mod settings; #[cfg(test)] mod test_utils; pub mod upgrade; @@ -54,7 +55,7 @@ use meilisearch_types::batches::Batch; use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures}; use meilisearch_types::heed::byteorder::BE; use meilisearch_types::heed::types::{SerdeJson, Str, I128}; -use meilisearch_types::heed::{self, Database, Env, RoTxn, WithoutTls}; +use meilisearch_types::heed::{self, Database, Env, RoTxn, Unspecified, WithoutTls}; use meilisearch_types::milli::index::IndexEmbeddingConfig; use meilisearch_types::milli::update::IndexerConfig; use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs}; @@ -142,6 +143,8 @@ pub struct IndexScheduler { /// The list of tasks currently processing pub(crate) processing_tasks: Arc>, + /// The main database that also has the chat settings. + pub main: Database, /// A database containing only the version of the index-scheduler pub version: versioning::Versioning, /// The queue containing both the tasks and the batches. @@ -151,9 +154,6 @@ pub struct IndexScheduler { /// In charge of fetching and setting the status of experimental features. features: features::FeatureData, - /// Stores the custom chat prompts and other settings of the indexes. - chat_settings: Database>, - /// Everything related to the processing of the tasks pub scheduler: scheduler::Scheduler, @@ -199,7 +199,7 @@ impl IndexScheduler { version: self.version.clone(), queue: self.queue.private_clone(), scheduler: self.scheduler.private_clone(), - + main: self.main.clone(), index_mapper: self.index_mapper.clone(), cleanup_enabled: self.cleanup_enabled, webhook_url: self.webhook_url.clone(), @@ -212,16 +212,11 @@ impl IndexScheduler { #[cfg(test)] run_loop_iteration: self.run_loop_iteration.clone(), features: self.features.clone(), - chat_settings: self.chat_settings, } } pub(crate) const fn nb_db() -> u32 { - Versioning::nb_db() - + Queue::nb_db() - + IndexMapper::nb_db() - + features::FeatureData::nb_db() - + 1 // chat-prompts + Versioning::nb_db() + Queue::nb_db() + IndexMapper::nb_db() + features::FeatureData::nb_db() } /// Create an index scheduler and start its run loop. diff --git a/crates/index-scheduler/src/settings/chat.rs b/crates/index-scheduler/src/settings/chat.rs new file mode 100644 index 000000000..bec1f4561 --- /dev/null +++ b/crates/index-scheduler/src/settings/chat.rs @@ -0,0 +1,432 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::convert::Infallible; +use std::fmt; +use std::marker::PhantomData; +use std::num::NonZeroUsize; +use std::ops::{ControlFlow, Deref}; +use std::str::FromStr; + +use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef}; +use fst::IntoStreamer; +use milli::disabled_typos_terms::DisabledTyposTerms; +use milli::index::{IndexEmbeddingConfig, PrefixSearch}; +use milli::proximity::ProximityPrecision; +use milli::update::Setting; +use milli::{FilterableAttributesRule, Index}; +use serde::{Deserialize, Serialize, Serializer}; +use utoipa::ToSchema; + +use crate::deserr::DeserrJsonError; +use crate::error::deserr_codes::*; +use crate::heed::RoTxn; +use crate::IndexScheduler; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[deserr(deny_unknown_fields, rename_all = camelCase, where_predicate = __Deserr_E: deserr::MergeWithError>)] +pub struct PromptsSettings { + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + pub system: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!({ "oneTypo": 5, "twoTypo": 9 }))] + pub search_description: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + pub search_q_param: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + pub pre_query: Setting, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub enum ChatSource { + #[default] + OpenAi, +} + +/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings +/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a +/// call to `check` will return a `Settings` from a `Settings`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] +#[serde( + deny_unknown_fields, + rename_all = "camelCase", + bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>") +)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[schema(rename_all = "camelCase")] +pub struct ChatSettings { + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option)] + pub source: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option)] + pub base_api: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option)] + pub api_key: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option)] + pub prompts: Setting, + + #[serde(skip)] + #[deserr(skip)] + pub _kind: PhantomData, +} + +impl ChatSettings { + pub fn hide_secrets(&mut self) { + match &mut self.api_key { + Setting::Set(key) => Self::hide_secrets(key), + Setting::Reset => todo!(), + Setting::NotSet => todo!(), + } + } + + fn hide_secret(secret: &mut String) { + match secret.len() { + x if x < 10 => { + secret.replace_range(.., "XXX..."); + } + x if x < 20 => { + secret.replace_range(2.., "XXXX..."); + } + x if x < 30 => { + secret.replace_range(3.., "XXXXX..."); + } + _x => { + secret.replace_range(5.., "XXXXXX..."); + } + } + } +} + +impl ChatSettings { + pub fn cleared() -> ChatSettings { + ChatSettings { + source: Setting::Reset, + base_api: Setting::Reset, + api_key: Setting::Reset, + prompts: Setting::Reset, + _kind: PhantomData, + } + } + + pub fn into_unchecked(self) -> ChatSettings { + let Self { source, base_api, api_key, prompts, _kind } = self; + ChatSettings { source, base_api, api_key, prompts, _kind: PhantomData } + } +} + +impl ChatSettings { + pub fn check(self) -> ChatSettings { + ChatSettings { + source: self.source, + base_api: self.base_api, + api_key: self.api_key, + prompts: self.prompts, + _kind: PhantomData, + } + } + + pub fn validate(self) -> Result { + self.validate_prompt_settings()?; + self.validate_global_settings() + } + + fn validate_global_settings(mut self) -> Result { + // Check that the ApiBase is a valid URL + Ok(self) + } + + fn validate_prompt_settings(mut self) -> Result { + // TODO + // let Setting::Set(mut configs) = self.embedders else { return Ok(self) }; + // for (name, config) in configs.iter_mut() { + // let config_to_check = std::mem::take(config); + // let checked_config = + // milli::update::validate_embedding_settings(config_to_check.inner, name)?; + // *config = SettingEmbeddingSettings { inner: checked_config }; + // } + // self.embedders = Setting::Set(configs); + Ok(self) + } + + pub fn merge(&mut self, other: &Self) { + // For most settings only the latest version is kept + *self = Self { + source: other.source.or(self.source), + base_api: other.base_api.or(self.base_api), + api_key: other.api_key.or(self.api_key), + prompts: match (self.prompts, other.prompts) { + (Setting::NotSet, set) | (set, Setting::NotSet) => set, + (Setting::Set(_) | Setting::Reset, Setting::Reset) => Setting::Reset, + (Setting::Reset, Setting::Set(set)) => Setting::Set(set), + // If both are set we must merge the prompts settings + (Setting::Set(this), Setting::Set(other)) => Setting::Set(PromptsSettings { + system: other.system.or(system), + search_description: other.search_description.or(search_description), + search_q_param: other.search_q_param.or(search_q_param), + pre_query: other.pre_query.or(pre_query), + }), + }, + + _kind: PhantomData, + } + } +} + +pub fn apply_settings_to_builder( + settings: &ChatSettings, + // TODO we must not store this into milli but in the index scheduler + builder: &mut milli::update::Settings, +) { + let ChatSettings { source, base_api, api_key, prompts, _kind } = settings; + + match source.deref() { + Setting::Set(ref names) => builder.set_searchable_fields(names.clone()), + Setting::Reset => builder.reset_searchable_fields(), + Setting::NotSet => (), + } + + match displayed_attributes.deref() { + Setting::Set(ref names) => builder.set_displayed_fields(names.clone()), + Setting::Reset => builder.reset_displayed_fields(), + Setting::NotSet => (), + } + + match filterable_attributes { + Setting::Set(ref facets) => { + builder.set_filterable_fields(facets.clone().into_iter().collect()) + } + Setting::Reset => builder.reset_filterable_fields(), + Setting::NotSet => (), + } + + match sortable_attributes { + Setting::Set(ref fields) => builder.set_sortable_fields(fields.iter().cloned().collect()), + Setting::Reset => builder.reset_sortable_fields(), + Setting::NotSet => (), + } + + match ranking_rules { + Setting::Set(ref criteria) => { + builder.set_criteria(criteria.iter().map(|c| c.clone().into()).collect()) + } + Setting::Reset => builder.reset_criteria(), + Setting::NotSet => (), + } + + match stop_words { + Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()), + Setting::Reset => builder.reset_stop_words(), + Setting::NotSet => (), + } + + match non_separator_tokens { + Setting::Set(ref non_separator_tokens) => { + builder.set_non_separator_tokens(non_separator_tokens.clone()) + } + Setting::Reset => builder.reset_non_separator_tokens(), + Setting::NotSet => (), + } + + match separator_tokens { + Setting::Set(ref separator_tokens) => { + builder.set_separator_tokens(separator_tokens.clone()) + } + Setting::Reset => builder.reset_separator_tokens(), + Setting::NotSet => (), + } + + match dictionary { + Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()), + Setting::Reset => builder.reset_dictionary(), + Setting::NotSet => (), + } + + match synonyms { + Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()), + Setting::Reset => builder.reset_synonyms(), + Setting::NotSet => (), + } + + match distinct_attribute { + Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()), + Setting::Reset => builder.reset_distinct_field(), + Setting::NotSet => (), + } + + match proximity_precision { + Setting::Set(ref precision) => builder.set_proximity_precision((*precision).into()), + Setting::Reset => builder.reset_proximity_precision(), + Setting::NotSet => (), + } + + match localized_attributes_rules { + Setting::Set(ref rules) => builder + .set_localized_attributes_rules(rules.iter().cloned().map(|r| r.into()).collect()), + Setting::Reset => builder.reset_localized_attributes_rules(), + Setting::NotSet => (), + } + + match typo_tolerance { + Setting::Set(ref value) => { + match value.enabled { + Setting::Set(val) => builder.set_autorize_typos(val), + Setting::Reset => builder.reset_authorize_typos(), + Setting::NotSet => (), + } + + match value.min_word_size_for_typos { + Setting::Set(ref setting) => { + match setting.one_typo { + Setting::Set(val) => builder.set_min_word_len_one_typo(val), + Setting::Reset => builder.reset_min_word_len_one_typo(), + Setting::NotSet => (), + } + match setting.two_typos { + Setting::Set(val) => builder.set_min_word_len_two_typos(val), + Setting::Reset => builder.reset_min_word_len_two_typos(), + Setting::NotSet => (), + } + } + Setting::Reset => { + builder.reset_min_word_len_one_typo(); + builder.reset_min_word_len_two_typos(); + } + Setting::NotSet => (), + } + + match value.disable_on_words { + Setting::Set(ref words) => { + builder.set_exact_words(words.clone()); + } + Setting::Reset => builder.reset_exact_words(), + Setting::NotSet => (), + } + + match value.disable_on_attributes { + Setting::Set(ref words) => { + builder.set_exact_attributes(words.iter().cloned().collect()) + } + Setting::Reset => builder.reset_exact_attributes(), + Setting::NotSet => (), + } + + match value.disable_on_numbers { + Setting::Set(val) => builder.set_disable_on_numbers(val), + Setting::Reset => builder.reset_disable_on_numbers(), + Setting::NotSet => (), + } + } + Setting::Reset => { + // all typo settings need to be reset here. + builder.reset_authorize_typos(); + builder.reset_min_word_len_one_typo(); + builder.reset_min_word_len_two_typos(); + builder.reset_exact_words(); + builder.reset_exact_attributes(); + } + Setting::NotSet => (), + } + + match faceting { + Setting::Set(FacetingSettings { max_values_per_facet, sort_facet_values_by }) => { + match max_values_per_facet { + Setting::Set(val) => builder.set_max_values_per_facet(*val), + Setting::Reset => builder.reset_max_values_per_facet(), + Setting::NotSet => (), + } + match sort_facet_values_by { + Setting::Set(val) => builder.set_sort_facet_values_by( + val.iter().map(|(name, order)| (name.clone(), (*order).into())).collect(), + ), + Setting::Reset => builder.reset_sort_facet_values_by(), + Setting::NotSet => (), + } + } + Setting::Reset => { + builder.reset_max_values_per_facet(); + builder.reset_sort_facet_values_by(); + } + Setting::NotSet => (), + } + + match pagination { + Setting::Set(ref value) => match value.max_total_hits { + Setting::Set(val) => builder.set_pagination_max_total_hits(val), + Setting::Reset => builder.reset_pagination_max_total_hits(), + Setting::NotSet => (), + }, + Setting::Reset => builder.reset_pagination_max_total_hits(), + Setting::NotSet => (), + } + + match embedders { + Setting::Set(value) => builder.set_embedder_settings( + value.iter().map(|(k, v)| (k.clone(), v.inner.clone())).collect(), + ), + Setting::Reset => builder.reset_embedder_settings(), + Setting::NotSet => (), + } + + match search_cutoff_ms { + Setting::Set(cutoff) => builder.set_search_cutoff(*cutoff), + Setting::Reset => builder.reset_search_cutoff(), + Setting::NotSet => (), + } + + match prefix_search { + Setting::Set(prefix_search) => { + builder.set_prefix_search(PrefixSearch::from(*prefix_search)) + } + Setting::Reset => builder.reset_prefix_search(), + Setting::NotSet => (), + } + + match facet_search { + Setting::Set(facet_search) => builder.set_facet_search(*facet_search), + Setting::Reset => builder.reset_facet_search(), + Setting::NotSet => (), + } + + match chat { + Setting::Set(chat) => builder.set_chat(chat.clone()), + Setting::Reset => builder.reset_chat(), + Setting::NotSet => (), + } +} + +pub enum SecretPolicy { + RevealSecrets, + HideSecrets, +} + +pub fn settings( + index_scheduler: &IndexScheduler, + rtxn: &RoTxn, + secret_policy: SecretPolicy, +) -> Result, milli::Error> { + let mut settings = index_scheduler.chat_settings(rtxn)?; + if let SecretPolicy::HideSecrets = secret_policy { + settings.hide_secrets() + } + Ok(settings) +} diff --git a/crates/index-scheduler/src/settings/mod.rs b/crates/index-scheduler/src/settings/mod.rs new file mode 100644 index 000000000..0801c64d0 --- /dev/null +++ b/crates/index-scheduler/src/settings/mod.rs @@ -0,0 +1,3 @@ +mod chat; + +pub use chat::ChatSettings; diff --git a/crates/meilisearch-types/src/settings.rs b/crates/meilisearch-types/src/settings.rs index 3f92c1c52..bd9757a5b 100644 --- a/crates/meilisearch-types/src/settings.rs +++ b/crates/meilisearch-types/src/settings.rs @@ -17,6 +17,7 @@ use milli::{Criterion, CriterionError, FilterableAttributesRule, Index, DEFAULT_ use serde::{Deserialize, Serialize, Serializer}; use utoipa::ToSchema; +use super::{Checked, Unchecked}; use crate::deserr::DeserrJsonError; use crate::error::deserr_codes::*; use crate::facet_values_sort::FacetValuesSort; diff --git a/crates/meilisearch/src/routes/settings/chat.rs b/crates/meilisearch/src/routes/settings/chat.rs index 586fa041e..6a0962e39 100644 --- a/crates/meilisearch/src/routes/settings/chat.rs +++ b/crates/meilisearch/src/routes/settings/chat.rs @@ -91,7 +91,7 @@ Selecting the right index ensures the most relevant results for the user query"; impl Default for GlobalChatSettings { fn default() -> Self { GlobalChatSettings { - source: "openai".to_string(), + source: "openAi".to_string(), base_api: None, api_key: None, prompts: ChatPrompts {