mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-06-06 04:05:37 +00:00
Implement useful conversion strategies and clean up the code
This commit is contained in:
parent
8c66e63760
commit
7e42293a23
@ -29,7 +29,7 @@ use meilisearch_auth::AuthController;
|
|||||||
use meilisearch_types::error::ResponseError;
|
use meilisearch_types::error::ResponseError;
|
||||||
use meilisearch_types::heed::RoTxn;
|
use meilisearch_types::heed::RoTxn;
|
||||||
use meilisearch_types::keys::actions;
|
use meilisearch_types::keys::actions;
|
||||||
use meilisearch_types::milli::index::{self, ChatConfig, SearchParameters};
|
use meilisearch_types::milli::index::ChatConfig;
|
||||||
use meilisearch_types::milli::prompt::{Prompt, PromptData};
|
use meilisearch_types::milli::prompt::{Prompt, PromptData};
|
||||||
use meilisearch_types::milli::update::new::document::DocumentFromDb;
|
use meilisearch_types::milli::update::new::document::DocumentFromDb;
|
||||||
use meilisearch_types::milli::update::Setting;
|
use meilisearch_types::milli::update::Setting;
|
||||||
@ -50,11 +50,7 @@ use crate::extractors::authentication::policies::ActionPolicy;
|
|||||||
use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _};
|
use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _};
|
||||||
use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
|
use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
|
||||||
use crate::routes::indexes::search::search_kind;
|
use crate::routes::indexes::search::search_kind;
|
||||||
use crate::search::{
|
use crate::search::{add_search_rules, prepare_search, search_from_kind, SearchQuery};
|
||||||
add_search_rules, prepare_search, search_from_kind, HybridQuery, MatchingStrategy,
|
|
||||||
RankingScoreThreshold, SearchQuery, SemanticRatio, DEFAULT_SEARCH_LIMIT,
|
|
||||||
DEFAULT_SEMANTIC_RATIO,
|
|
||||||
};
|
|
||||||
use crate::search_queue::SearchQueue;
|
use crate::search_queue::SearchQueue;
|
||||||
|
|
||||||
const MEILI_SEARCH_PROGRESS_NAME: &str = "_meiliSearchProgress";
|
const MEILI_SEARCH_PROGRESS_NAME: &str = "_meiliSearchProgress";
|
||||||
@ -228,40 +224,7 @@ async fn process_search_request(
|
|||||||
let index = index_scheduler.index(&index_uid)?;
|
let index = index_scheduler.index(&index_uid)?;
|
||||||
let rtxn = index.static_read_txn()?;
|
let rtxn = index.static_read_txn()?;
|
||||||
let ChatConfig { description: _, prompt: _, search_parameters } = index.chat_config(&rtxn)?;
|
let ChatConfig { description: _, prompt: _, search_parameters } = index.chat_config(&rtxn)?;
|
||||||
let SearchParameters {
|
let mut query = SearchQuery { q, ..SearchQuery::from(search_parameters) };
|
||||||
hybrid,
|
|
||||||
limit,
|
|
||||||
sort,
|
|
||||||
distinct,
|
|
||||||
matching_strategy,
|
|
||||||
attributes_to_search_on,
|
|
||||||
ranking_score_threshold,
|
|
||||||
} = search_parameters;
|
|
||||||
|
|
||||||
let mut query = SearchQuery {
|
|
||||||
q,
|
|
||||||
hybrid: hybrid.map(|index::HybridQuery { semantic_ratio, embedder }| HybridQuery {
|
|
||||||
semantic_ratio: SemanticRatio::try_from(semantic_ratio)
|
|
||||||
.ok()
|
|
||||||
.unwrap_or_else(DEFAULT_SEMANTIC_RATIO),
|
|
||||||
embedder,
|
|
||||||
}),
|
|
||||||
limit: limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
|
|
||||||
sort,
|
|
||||||
distinct,
|
|
||||||
matching_strategy: matching_strategy
|
|
||||||
.map(|ms| match ms {
|
|
||||||
index::MatchingStrategy::Last => MatchingStrategy::Last,
|
|
||||||
index::MatchingStrategy::All => MatchingStrategy::All,
|
|
||||||
index::MatchingStrategy::Frequency => MatchingStrategy::Frequency,
|
|
||||||
})
|
|
||||||
.unwrap_or(MatchingStrategy::Frequency),
|
|
||||||
attributes_to_search_on,
|
|
||||||
ranking_score_threshold: ranking_score_threshold
|
|
||||||
.and_then(|rst| RankingScoreThreshold::try_from(rst).ok()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
let auth_filter = ActionPolicy::<{ actions::SEARCH }>::authenticate(
|
let auth_filter = ActionPolicy::<{ actions::SEARCH }>::authenticate(
|
||||||
auth_ctrl,
|
auth_ctrl,
|
||||||
auth_token,
|
auth_token,
|
||||||
|
@ -16,6 +16,7 @@ use meilisearch_types::error::{Code, ResponseError};
|
|||||||
use meilisearch_types::heed::RoTxn;
|
use meilisearch_types::heed::RoTxn;
|
||||||
use meilisearch_types::index_uid::IndexUid;
|
use meilisearch_types::index_uid::IndexUid;
|
||||||
use meilisearch_types::locales::Locale;
|
use meilisearch_types::locales::Locale;
|
||||||
|
use meilisearch_types::milli::index::{self, SearchParameters};
|
||||||
use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
|
use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
|
||||||
use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
|
use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
|
||||||
use meilisearch_types::milli::vector::Embedder;
|
use meilisearch_types::milli::vector::Embedder;
|
||||||
@ -56,7 +57,7 @@ pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "<em>".to_string();
|
|||||||
pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
|
pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
|
||||||
pub const DEFAULT_SEMANTIC_RATIO: fn() -> SemanticRatio = || SemanticRatio(0.5);
|
pub const DEFAULT_SEMANTIC_RATIO: fn() -> SemanticRatio = || SemanticRatio(0.5);
|
||||||
|
|
||||||
#[derive(Clone, Default, PartialEq, Deserr, ToSchema)]
|
#[derive(Clone, PartialEq, Deserr, ToSchema)]
|
||||||
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
|
#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
|
||||||
pub struct SearchQuery {
|
pub struct SearchQuery {
|
||||||
#[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
|
#[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
|
||||||
@ -119,6 +120,69 @@ pub struct SearchQuery {
|
|||||||
pub locales: Option<Vec<Locale>>,
|
pub locales: Option<Vec<Locale>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<SearchParameters> for SearchQuery {
|
||||||
|
fn from(parameters: SearchParameters) -> Self {
|
||||||
|
let SearchParameters {
|
||||||
|
hybrid,
|
||||||
|
limit,
|
||||||
|
sort,
|
||||||
|
distinct,
|
||||||
|
matching_strategy,
|
||||||
|
attributes_to_search_on,
|
||||||
|
ranking_score_threshold,
|
||||||
|
} = parameters;
|
||||||
|
|
||||||
|
SearchQuery {
|
||||||
|
hybrid: hybrid.map(|index::HybridQuery { semantic_ratio, embedder }| HybridQuery {
|
||||||
|
semantic_ratio: SemanticRatio::try_from(semantic_ratio)
|
||||||
|
.ok()
|
||||||
|
.unwrap_or_else(DEFAULT_SEMANTIC_RATIO),
|
||||||
|
embedder,
|
||||||
|
}),
|
||||||
|
limit: limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
|
||||||
|
sort,
|
||||||
|
distinct,
|
||||||
|
matching_strategy: matching_strategy.map(MatchingStrategy::from).unwrap_or_default(),
|
||||||
|
attributes_to_search_on,
|
||||||
|
ranking_score_threshold: ranking_score_threshold.map(RankingScoreThreshold::from),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SearchQuery {
|
||||||
|
fn default() -> Self {
|
||||||
|
SearchQuery {
|
||||||
|
q: None,
|
||||||
|
vector: None,
|
||||||
|
hybrid: None,
|
||||||
|
offset: DEFAULT_SEARCH_OFFSET(),
|
||||||
|
limit: DEFAULT_SEARCH_LIMIT(),
|
||||||
|
page: None,
|
||||||
|
hits_per_page: None,
|
||||||
|
attributes_to_retrieve: None,
|
||||||
|
retrieve_vectors: false,
|
||||||
|
attributes_to_crop: None,
|
||||||
|
crop_length: DEFAULT_CROP_LENGTH(),
|
||||||
|
attributes_to_highlight: None,
|
||||||
|
show_matches_position: false,
|
||||||
|
show_ranking_score: false,
|
||||||
|
show_ranking_score_details: false,
|
||||||
|
filter: None,
|
||||||
|
sort: None,
|
||||||
|
distinct: None,
|
||||||
|
facets: None,
|
||||||
|
highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(),
|
||||||
|
highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),
|
||||||
|
crop_marker: DEFAULT_CROP_MARKER(),
|
||||||
|
matching_strategy: Default::default(),
|
||||||
|
attributes_to_search_on: None,
|
||||||
|
ranking_score_threshold: None,
|
||||||
|
locales: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Deserr, ToSchema, Serialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Deserr, ToSchema, Serialize)]
|
||||||
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
|
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
|
||||||
pub struct RankingScoreThreshold(f64);
|
pub struct RankingScoreThreshold(f64);
|
||||||
@ -137,6 +201,14 @@ impl std::convert::TryFrom<f64> for RankingScoreThreshold {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<index::RankingScoreThreshold> for RankingScoreThreshold {
|
||||||
|
fn from(threshold: index::RankingScoreThreshold) -> Self {
|
||||||
|
let threshold = threshold.as_f64();
|
||||||
|
assert!(threshold >= 0.0 && threshold <= 1.0);
|
||||||
|
RankingScoreThreshold(threshold)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Deserr)]
|
#[derive(Debug, Clone, Copy, PartialEq, Deserr)]
|
||||||
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSimilarRankingScoreThreshold)]
|
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSimilarRankingScoreThreshold)]
|
||||||
pub struct RankingScoreThresholdSimilar(f64);
|
pub struct RankingScoreThresholdSimilar(f64);
|
||||||
@ -718,6 +790,16 @@ impl From<MatchingStrategy> for TermsMatchingStrategy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<index::MatchingStrategy> for MatchingStrategy {
|
||||||
|
fn from(other: index::MatchingStrategy) -> Self {
|
||||||
|
match other {
|
||||||
|
index::MatchingStrategy::Last => Self::Last,
|
||||||
|
index::MatchingStrategy::All => Self::All,
|
||||||
|
index::MatchingStrategy::Frequency => Self::Frequency,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Deserr)]
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Deserr)]
|
||||||
#[deserr(rename_all = camelCase)]
|
#[deserr(rename_all = camelCase)]
|
||||||
pub enum FacetValuesSort {
|
pub enum FacetValuesSort {
|
||||||
@ -1261,7 +1343,7 @@ struct HitMaker<'a> {
|
|||||||
vectors_fid: Option<FieldId>,
|
vectors_fid: Option<FieldId>,
|
||||||
retrieve_vectors: RetrieveVectors,
|
retrieve_vectors: RetrieveVectors,
|
||||||
to_retrieve_ids: BTreeSet<FieldId>,
|
to_retrieve_ids: BTreeSet<FieldId>,
|
||||||
embedding_configs: Vec<milli::index::IndexEmbeddingConfig>,
|
embedding_configs: Vec<index::IndexEmbeddingConfig>,
|
||||||
formatter_builder: MatcherBuilder<'a>,
|
formatter_builder: MatcherBuilder<'a>,
|
||||||
formatted_options: BTreeMap<FieldId, FormatOptions>,
|
formatted_options: BTreeMap<FieldId, FormatOptions>,
|
||||||
show_ranking_score: bool,
|
show_ranking_score: bool,
|
||||||
|
@ -1,14 +1,18 @@
|
|||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||||
|
use std::error::Error;
|
||||||
|
use std::fmt;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
|
use deserr::Deserr;
|
||||||
use heed::types::*;
|
use heed::types::*;
|
||||||
use heed::{CompactionOption, Database, DatabaseStat, RoTxn, RwTxn, Unspecified, WithoutTls};
|
use heed::{CompactionOption, Database, DatabaseStat, RoTxn, RwTxn, Unspecified, WithoutTls};
|
||||||
use indexmap::IndexMap;
|
use indexmap::IndexMap;
|
||||||
use roaring::RoaringBitmap;
|
use roaring::RoaringBitmap;
|
||||||
use rstar::RTree;
|
use rstar::RTree;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::constants::{self, RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME};
|
use crate::constants::{self, RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME};
|
||||||
use crate::database_stats::DatabaseStats;
|
use crate::database_stats::DatabaseStats;
|
||||||
@ -25,6 +29,7 @@ use crate::heed_codec::{BEU16StrCodec, FstSetCodec, StrBEU16Codec, StrRefCodec};
|
|||||||
use crate::order_by_map::OrderByMap;
|
use crate::order_by_map::OrderByMap;
|
||||||
use crate::prompt::PromptData;
|
use crate::prompt::PromptData;
|
||||||
use crate::proximity::ProximityPrecision;
|
use crate::proximity::ProximityPrecision;
|
||||||
|
use crate::update::new::StdResult;
|
||||||
use crate::vector::{ArroyStats, ArroyWrapper, Embedding, EmbeddingConfig};
|
use crate::vector::{ArroyStats, ArroyWrapper, Embedding, EmbeddingConfig};
|
||||||
use crate::{
|
use crate::{
|
||||||
default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds,
|
default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds,
|
||||||
@ -1962,10 +1967,46 @@ pub struct SearchParameters {
|
|||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub attributes_to_search_on: Option<Vec<String>>,
|
pub attributes_to_search_on: Option<Vec<String>>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub ranking_score_threshold: Option<f64>,
|
pub ranking_score_threshold: Option<RankingScoreThreshold>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Deserialize, Serialize)]
|
#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Deserr, ToSchema)]
|
||||||
|
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
|
||||||
|
pub struct RankingScoreThreshold(f64);
|
||||||
|
|
||||||
|
impl RankingScoreThreshold {
|
||||||
|
pub fn as_f64(&self) -> f64 {
|
||||||
|
self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TryFrom<f64> for RankingScoreThreshold {
|
||||||
|
type Error = InvalidSearchRankingScoreThreshold;
|
||||||
|
|
||||||
|
fn try_from(value: f64) -> StdResult<Self, Self::Error> {
|
||||||
|
if value < 0.0 || value > 1.0 {
|
||||||
|
Err(InvalidSearchRankingScoreThreshold)
|
||||||
|
} else {
|
||||||
|
Ok(RankingScoreThreshold(value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct InvalidSearchRankingScoreThreshold;
|
||||||
|
|
||||||
|
impl Error for InvalidSearchRankingScoreThreshold {}
|
||||||
|
|
||||||
|
impl fmt::Display for InvalidSearchRankingScoreThreshold {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"the value of `rankingScoreThreshold` is invalid, expected a float between `0.0` and `1.0`."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct HybridQuery {
|
pub struct HybridQuery {
|
||||||
pub semantic_ratio: f32,
|
pub semantic_ratio: f32,
|
||||||
@ -1980,10 +2021,12 @@ pub struct PrefixSettings {
|
|||||||
pub compute_prefixes: PrefixSearch,
|
pub compute_prefixes: PrefixSearch,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Deserr, ToSchema, Serialize, Deserialize)]
|
||||||
|
#[deserr(rename_all = camelCase)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub enum MatchingStrategy {
|
pub enum MatchingStrategy {
|
||||||
/// Remove query words from last to first
|
/// Remove query words from last to first
|
||||||
|
#[default]
|
||||||
Last,
|
Last,
|
||||||
/// All query words are mandatory
|
/// All query words are mandatory
|
||||||
All,
|
All,
|
||||||
|
@ -64,7 +64,7 @@ fn default_template() -> liquid::Template {
|
|||||||
new_template(default_template_text()).unwrap()
|
new_template(default_template_text()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_template_text() -> &'static str {
|
pub fn default_template_text() -> &'static str {
|
||||||
"{% for field in fields %}\
|
"{% for field in fields %}\
|
||||||
{% if field.is_searchable and field.value != nil %}\
|
{% if field.is_searchable and field.value != nil %}\
|
||||||
{{ field.name }}: {{ field.value }}\n\
|
{{ field.name }}: {{ field.value }}\n\
|
||||||
|
@ -10,6 +10,7 @@ pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FAC
|
|||||||
pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords};
|
pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords};
|
||||||
use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats};
|
use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats};
|
||||||
use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features};
|
use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features};
|
||||||
|
use crate::index::MatchingStrategy;
|
||||||
use crate::score_details::{ScoreDetails, ScoringStrategy};
|
use crate::score_details::{ScoreDetails, ScoringStrategy};
|
||||||
use crate::vector::Embedder;
|
use crate::vector::Embedder;
|
||||||
use crate::{
|
use crate::{
|
||||||
@ -364,6 +365,16 @@ impl Default for TermsMatchingStrategy {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<MatchingStrategy> for TermsMatchingStrategy {
|
||||||
|
fn from(other: MatchingStrategy) -> Self {
|
||||||
|
match other {
|
||||||
|
MatchingStrategy::Last => Self::Last,
|
||||||
|
MatchingStrategy::All => Self::All,
|
||||||
|
MatchingStrategy::Frequency => Self::Frequency,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn get_first(s: &str) -> &str {
|
fn get_first(s: &str) -> &str {
|
||||||
match s.chars().next() {
|
match s.chars().next() {
|
||||||
Some(c) => &s[..c.len_utf8()],
|
Some(c) => &s[..c.len_utf8()],
|
||||||
|
@ -6,10 +6,9 @@ use deserr::Deserr;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use utoipa::ToSchema;
|
use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::index::{self, ChatConfig, SearchParameters};
|
use crate::index::{self, ChatConfig, MatchingStrategy, RankingScoreThreshold, SearchParameters};
|
||||||
use crate::prompt::{default_max_bytes, PromptData};
|
use crate::prompt::{default_max_bytes, PromptData};
|
||||||
use crate::update::Setting;
|
use crate::update::Setting;
|
||||||
use crate::TermsMatchingStrategy;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Deserr, ToSchema)]
|
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Deserr, ToSchema)]
|
||||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||||
@ -70,13 +69,10 @@ impl From<ChatConfig> for ChatSettings {
|
|||||||
HybridQuery { semantic_ratio: SemanticRatio(semantic_ratio), embedder }
|
HybridQuery { semantic_ratio: SemanticRatio(semantic_ratio), embedder }
|
||||||
});
|
});
|
||||||
|
|
||||||
let matching_strategy = matching_strategy.map(|ms| match ms {
|
let matching_strategy = matching_strategy.map(MatchingStrategy::from);
|
||||||
index::MatchingStrategy::Last => MatchingStrategy::Last,
|
|
||||||
index::MatchingStrategy::All => MatchingStrategy::All,
|
|
||||||
index::MatchingStrategy::Frequency => MatchingStrategy::Frequency,
|
|
||||||
});
|
|
||||||
|
|
||||||
let ranking_score_threshold = ranking_score_threshold.map(RankingScoreThreshold);
|
let ranking_score_threshold =
|
||||||
|
ranking_score_threshold.map(RankingScoreThreshold::from);
|
||||||
|
|
||||||
ChatSearchParams {
|
ChatSearchParams {
|
||||||
hybrid: Setting::some_or_not_set(hybrid),
|
hybrid: Setting::some_or_not_set(hybrid),
|
||||||
@ -197,63 +193,3 @@ impl std::ops::Deref for SemanticRatio {
|
|||||||
&self.0
|
&self.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, ToSchema, Serialize, Deserialize)]
|
|
||||||
#[deserr(rename_all = camelCase)]
|
|
||||||
#[serde(rename_all = "camelCase")]
|
|
||||||
pub enum MatchingStrategy {
|
|
||||||
/// Remove query words from last to first
|
|
||||||
Last,
|
|
||||||
/// All query words are mandatory
|
|
||||||
All,
|
|
||||||
/// Remove query words from the most frequent to the least
|
|
||||||
Frequency,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for MatchingStrategy {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::Last
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<MatchingStrategy> for TermsMatchingStrategy {
|
|
||||||
fn from(other: MatchingStrategy) -> Self {
|
|
||||||
match other {
|
|
||||||
MatchingStrategy::Last => Self::Last,
|
|
||||||
MatchingStrategy::All => Self::All,
|
|
||||||
MatchingStrategy::Frequency => Self::Frequency,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Deserr, ToSchema, Serialize, Deserialize)]
|
|
||||||
#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
|
|
||||||
pub struct RankingScoreThreshold(pub f64);
|
|
||||||
|
|
||||||
impl std::convert::TryFrom<f64> for RankingScoreThreshold {
|
|
||||||
type Error = InvalidSearchRankingScoreThreshold;
|
|
||||||
|
|
||||||
fn try_from(f: f64) -> Result<Self, Self::Error> {
|
|
||||||
// the suggested "fix" is: `!(0.0..=1.0).contains(&f)`` which is allegedly less readable
|
|
||||||
#[allow(clippy::manual_range_contains)]
|
|
||||||
if f > 1.0 || f < 0.0 {
|
|
||||||
Err(InvalidSearchRankingScoreThreshold)
|
|
||||||
} else {
|
|
||||||
Ok(RankingScoreThreshold(f))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct InvalidSearchRankingScoreThreshold;
|
|
||||||
|
|
||||||
impl Error for InvalidSearchRankingScoreThreshold {}
|
|
||||||
|
|
||||||
impl fmt::Display for InvalidSearchRankingScoreThreshold {
|
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
write!(
|
|
||||||
f,
|
|
||||||
"the value of `rankingScoreThreshold` is invalid, expected a float between `0.0` and `1.0`."
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
@ -11,7 +11,7 @@ use roaring::RoaringBitmap;
|
|||||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
use super::chat::{ChatSearchParams, RankingScoreThreshold};
|
use super::chat::ChatSearchParams;
|
||||||
use super::del_add::{DelAdd, DelAddOperation};
|
use super::del_add::{DelAdd, DelAddOperation};
|
||||||
use super::index_documents::{IndexDocumentsConfig, Transform};
|
use super::index_documents::{IndexDocumentsConfig, Transform};
|
||||||
use super::{ChatSettings, IndexerConfig};
|
use super::{ChatSettings, IndexerConfig};
|
||||||
@ -23,11 +23,11 @@ use crate::error::UserError;
|
|||||||
use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder};
|
use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder};
|
||||||
use crate::filterable_attributes_rules::match_faceted_field;
|
use crate::filterable_attributes_rules::match_faceted_field;
|
||||||
use crate::index::{
|
use crate::index::{
|
||||||
ChatConfig, IndexEmbeddingConfig, MatchingStrategy, PrefixSearch,
|
ChatConfig, IndexEmbeddingConfig, MatchingStrategy, PrefixSearch, RankingScoreThreshold,
|
||||||
DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS,
|
SearchParameters, DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS,
|
||||||
};
|
};
|
||||||
use crate::order_by_map::OrderByMap;
|
use crate::order_by_map::OrderByMap;
|
||||||
use crate::prompt::{default_max_bytes, PromptData};
|
use crate::prompt::{default_max_bytes, default_template_text, PromptData};
|
||||||
use crate::proximity::ProximityPrecision;
|
use crate::proximity::ProximityPrecision;
|
||||||
use crate::update::index_documents::IndexDocumentsMethod;
|
use crate::update::index_documents::IndexDocumentsMethod;
|
||||||
use crate::update::{IndexDocuments, UpdateIndexingStep};
|
use crate::update::{IndexDocuments, UpdateIndexingStep};
|
||||||
@ -1266,32 +1266,29 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
document_template_max_bytes: new_document_template_max_bytes,
|
document_template_max_bytes: new_document_template_max_bytes,
|
||||||
search_parameters: new_search_parameters,
|
search_parameters: new_search_parameters,
|
||||||
}) => {
|
}) => {
|
||||||
let mut old = self.index.chat_config(self.wtxn)?;
|
let ChatConfig { description, prompt, search_parameters } =
|
||||||
let ChatConfig {
|
self.index.chat_config(self.wtxn)?;
|
||||||
ref mut description,
|
|
||||||
prompt: PromptData { ref mut template, ref mut max_bytes },
|
|
||||||
ref mut search_parameters,
|
|
||||||
} = old;
|
|
||||||
|
|
||||||
match new_description {
|
let description = match new_description {
|
||||||
Setting::Set(d) => *description = d.clone(),
|
Setting::Set(new) => new.clone(),
|
||||||
Setting::Reset => *description = Default::default(),
|
Setting::Reset => Default::default(),
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => description,
|
||||||
}
|
};
|
||||||
|
|
||||||
match new_document_template {
|
let prompt = PromptData {
|
||||||
Setting::Set(dt) => *template = dt.clone(),
|
template: match new_document_template {
|
||||||
Setting::Reset => *template = Default::default(),
|
Setting::Set(new) => new.clone(),
|
||||||
Setting::NotSet => (),
|
Setting::Reset => default_template_text().to_string(),
|
||||||
}
|
Setting::NotSet => prompt.template.clone(),
|
||||||
|
},
|
||||||
|
max_bytes: match new_document_template_max_bytes {
|
||||||
|
Setting::Set(m) => NonZeroUsize::new(*m),
|
||||||
|
Setting::Reset => Some(default_max_bytes()),
|
||||||
|
Setting::NotSet => prompt.max_bytes,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
match new_document_template_max_bytes {
|
let search_parameters = match new_search_parameters {
|
||||||
Setting::Set(m) => *max_bytes = NonZeroUsize::new(*m),
|
|
||||||
Setting::Reset => *max_bytes = Some(default_max_bytes()),
|
|
||||||
Setting::NotSet => (),
|
|
||||||
}
|
|
||||||
|
|
||||||
match new_search_parameters {
|
|
||||||
Setting::Set(sp) => {
|
Setting::Set(sp) => {
|
||||||
let ChatSearchParams {
|
let ChatSearchParams {
|
||||||
hybrid,
|
hybrid,
|
||||||
@ -1303,74 +1300,62 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
|
|||||||
ranking_score_threshold,
|
ranking_score_threshold,
|
||||||
} = sp;
|
} = sp;
|
||||||
|
|
||||||
match hybrid {
|
SearchParameters {
|
||||||
Setting::Set(hybrid) => {
|
hybrid: match hybrid {
|
||||||
search_parameters.hybrid = Some(crate::index::HybridQuery {
|
Setting::Set(hybrid) => Some(crate::index::HybridQuery {
|
||||||
semantic_ratio: *hybrid.semantic_ratio,
|
semantic_ratio: *hybrid.semantic_ratio,
|
||||||
embedder: hybrid.embedder.clone(),
|
embedder: hybrid.embedder.clone(),
|
||||||
})
|
}),
|
||||||
}
|
Setting::Reset => None,
|
||||||
Setting::Reset => search_parameters.hybrid = None,
|
Setting::NotSet => search_parameters.hybrid.clone(),
|
||||||
Setting::NotSet => (),
|
},
|
||||||
}
|
limit: match limit {
|
||||||
|
Setting::Set(limit) => Some(*limit),
|
||||||
match limit {
|
Setting::Reset => None,
|
||||||
Setting::Set(limit) => search_parameters.limit = Some(*limit),
|
Setting::NotSet => search_parameters.limit,
|
||||||
Setting::Reset => search_parameters.limit = None,
|
},
|
||||||
Setting::NotSet => (),
|
sort: match sort {
|
||||||
}
|
Setting::Set(sort) => Some(sort.clone()),
|
||||||
|
Setting::Reset => None,
|
||||||
match sort {
|
Setting::NotSet => search_parameters.sort.clone(),
|
||||||
Setting::Set(sort) => search_parameters.sort = Some(sort.clone()),
|
},
|
||||||
Setting::Reset => search_parameters.sort = None,
|
distinct: match distinct {
|
||||||
Setting::NotSet => (),
|
Setting::Set(distinct) => Some(distinct.clone()),
|
||||||
}
|
Setting::Reset => None,
|
||||||
|
Setting::NotSet => search_parameters.distinct.clone(),
|
||||||
match distinct {
|
},
|
||||||
Setting::Set(distinct) => {
|
matching_strategy: match matching_strategy {
|
||||||
search_parameters.distinct = Some(distinct.clone())
|
Setting::Set(matching_strategy) => {
|
||||||
}
|
Some(MatchingStrategy::from(*matching_strategy))
|
||||||
Setting::Reset => search_parameters.distinct = None,
|
}
|
||||||
Setting::NotSet => (),
|
Setting::Reset => None,
|
||||||
}
|
Setting::NotSet => search_parameters.matching_strategy,
|
||||||
|
},
|
||||||
match matching_strategy {
|
attributes_to_search_on: match attributes_to_search_on {
|
||||||
Setting::Set(matching_strategy) => {
|
Setting::Set(attributes_to_search_on) => {
|
||||||
let strategy = match matching_strategy {
|
|
||||||
super::chat::MatchingStrategy::Last => MatchingStrategy::Last,
|
|
||||||
super::chat::MatchingStrategy::All => MatchingStrategy::All,
|
|
||||||
super::chat::MatchingStrategy::Frequency => {
|
|
||||||
MatchingStrategy::Frequency
|
|
||||||
}
|
|
||||||
};
|
|
||||||
search_parameters.matching_strategy = Some(strategy)
|
|
||||||
}
|
|
||||||
Setting::Reset => search_parameters.matching_strategy = None,
|
|
||||||
Setting::NotSet => (),
|
|
||||||
}
|
|
||||||
|
|
||||||
match attributes_to_search_on {
|
|
||||||
Setting::Set(attributes_to_search_on) => {
|
|
||||||
search_parameters.attributes_to_search_on =
|
|
||||||
Some(attributes_to_search_on.clone())
|
Some(attributes_to_search_on.clone())
|
||||||
}
|
}
|
||||||
Setting::Reset => search_parameters.attributes_to_search_on = None,
|
Setting::Reset => None,
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => {
|
||||||
}
|
search_parameters.attributes_to_search_on.clone()
|
||||||
|
}
|
||||||
match ranking_score_threshold {
|
},
|
||||||
Setting::Set(RankingScoreThreshold(score)) => {
|
ranking_score_threshold: match ranking_score_threshold {
|
||||||
search_parameters.ranking_score_threshold = Some(*score)
|
Setting::Set(rst) => Some(RankingScoreThreshold::from(*rst)),
|
||||||
}
|
Setting::Reset => None,
|
||||||
Setting::Reset => search_parameters.ranking_score_threshold = None,
|
Setting::NotSet => search_parameters.ranking_score_threshold,
|
||||||
Setting::NotSet => (),
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Setting::Reset => *search_parameters = Default::default(),
|
Setting::Reset => Default::default(),
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => search_parameters,
|
||||||
}
|
};
|
||||||
|
|
||||||
|
self.index.put_chat_config(
|
||||||
|
self.wtxn,
|
||||||
|
&ChatConfig { description, prompt, search_parameters },
|
||||||
|
)?;
|
||||||
|
|
||||||
self.index.put_chat_config(self.wtxn, &old)?;
|
|
||||||
Ok(true)
|
Ok(true)
|
||||||
}
|
}
|
||||||
Setting::Reset => self.index.delete_chat_config(self.wtxn),
|
Setting::Reset => self.index.delete_chat_config(self.wtxn),
|
||||||
|
@ -926,6 +926,7 @@ fn test_correct_settings_init() {
|
|||||||
assert!(matches!(prefix_search, Setting::NotSet));
|
assert!(matches!(prefix_search, Setting::NotSet));
|
||||||
assert!(matches!(facet_search, Setting::NotSet));
|
assert!(matches!(facet_search, Setting::NotSet));
|
||||||
assert!(matches!(disable_on_numbers, Setting::NotSet));
|
assert!(matches!(disable_on_numbers, Setting::NotSet));
|
||||||
|
assert!(matches!(chat, Setting::NotSet));
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user