Expose rankingScoreThreshold in API

4619: Use http path pattern instead of full path in metrics r=irevoire a=gh2k

# Pull Request

## Related issue

Fixes #3983 

## What does this PR do?

- This records only the HTTP pattern in metrics instead of the full path

An alternative solution was proposed in #4145, but this doesn't really fix the root cause of the issue. The problem I'm experiencing at my end is that by using the full path, the number of labels is far too high to be useful. It is normal practice to use the path with variable placeholders, instead of the fully-expanded path.

The example given in the ticket was endpoints under `/tasks`, but this can also be a very significant problem under `/indexes/{index-uid}/documents`. e.g.:
<img width="1510" alt="Screenshot 2024-05-03 at 12 14 36" src="https://github.com/meilisearch/meilisearch/assets/6530014/1df2ec19-5f69-4164-90d2-f65c59f9b544">

This patch replaces the fully-expanded path with the matched pattern.

The linked PR also mentions paths under other routes, e.g. `/static`, but this feels like a separate concern and these can be stripped out at the Prometheus end by filters if they are unwanted. The most important thing is to make the paths usable so that we can still get stats on e.g. the number of document deletes we see.

## PR checklist

Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Simon Detheridge <s@sd.ai>
Co-authored-by: Tamo <tamo@meilisearch.com>

Cargo.lock

@@ -2169,9 +2169,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "grenad"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a007932af5475ebb5c63bef8812bb1c36f317983bb4ca663e9d6dd58d6a0f8c"
+checksum = "c297f45167e6d543eb728e12ff284283e4ba2182a25c6cdcec883fda3316c7e7"
 dependencies = [
  "bytemuck",
  "byteorder",
@@ -2181,9 +2181,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.24"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
+checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
 dependencies = [
  "bytes",
  "fnv",

Cargo.toml

@@ -17,7 +17,8 @@ members = [
     "benchmarks",
     "fuzzers",
     "tracing-trace",
-    "xtask", "build-info",
+    "xtask",
+    "build-info",
 ]
 
 [workspace.package]

filter-parser/src/lib.rs

@@ -568,7 +568,7 @@ pub mod tests {
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
         insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequencies
+        // but it also works with other sequences
         insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
     }
 

index-scheduler/src/batch.rs

@@ -13,7 +13,7 @@ We can combine the two tasks in a single batch:
 1. import documents X and Y
 
 Processing this batch is functionally equivalent to processing the two
-tasks individally, but should be much faster since we are only performing
+tasks individually, but should be much faster since we are only performing
 one indexing operation.
 */
 

meilisearch-types/src/deserr/mod.rs

@@ -26,7 +26,7 @@ pub type DeserrQueryParamError<C = BadRequest> = DeserrError<DeserrQueryParam, C
 
 /// A request deserialization error.
 ///
-/// The first generic paramater is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
+/// The first generic parameter is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
 /// The second generic parameter is the default error code for the deserialization error, in case it is not given.
 pub struct DeserrError<Format, C: Default + ErrorCode> {
     pub msg: String,
@@ -189,3 +189,4 @@ merge_with_error_impl_take_error_message!(ParseTaskKindError);
 merge_with_error_impl_take_error_message!(ParseTaskStatusError);
 merge_with_error_impl_take_error_message!(IndexUidFormatError);
 merge_with_error_impl_take_error_message!(InvalidSearchSemanticRatio);
+merge_with_error_impl_take_error_message!(InvalidSearchRankingScoreThreshold);

meilisearch-types/src/error.rs

@@ -240,14 +240,12 @@ InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToRetrieve     , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRankingScoreThreshold    , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSemanticRatio            , InvalidRequest       , BAD_REQUEST ;
 InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
-InvalidRecommendContext               , InvalidRequest       , BAD_REQUEST ;
-InvalidRecommendId                    , InvalidRequest       , BAD_REQUEST ;
-InvalidRecommendPrompt                , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPostTag         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPreTag          , InvalidRequest       , BAD_REQUEST ;
@@ -311,8 +309,6 @@ MissingFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
 MissingIndexUid                       , InvalidRequest       , BAD_REQUEST ;
 MissingMasterKey                      , Auth                 , UNAUTHORIZED ;
 MissingPayload                        , InvalidRequest       , BAD_REQUEST ;
-MissingPrompt                         , InvalidRequest       , BAD_REQUEST ;
-MissingPromptOrId                     , InvalidRequest       , BAD_REQUEST ;
 MissingSearchHybrid                   , InvalidRequest       , BAD_REQUEST ;
 MissingSwapIndexes                    , InvalidRequest       , BAD_REQUEST ;
 MissingTaskFilters                    , InvalidRequest       , BAD_REQUEST ;
@@ -493,6 +489,15 @@ impl fmt::Display for deserr_codes::InvalidSearchSemanticRatio {
     }
 }
 
+impl fmt::Display for deserr_codes::InvalidSearchRankingScoreThreshold {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "the value of `rankingScoreThreshold` is invalid, expected a float between `0.0` and `1.0`."
+        )
+    }
+}
+
 #[macro_export]
 macro_rules! internal_error {
     ($target:ty : $($other:path), *) => {

meilisearch/src/analytics/segment_analytics.rs

@@ -672,6 +672,7 @@ impl SearchAggregator {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = query;
 
         let mut ret = Self::default();
@@ -1083,6 +1084,7 @@ impl MultiSearchAggregator {
                     matching_strategy: _,
                     attributes_to_search_on: _,
                     hybrid: _,
+                    ranking_score_threshold: _,
                 } = query;
 
                 index_uid.as_str()
@@ -1230,6 +1232,7 @@ impl FacetSearchAggregator {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = query;
 
         let mut ret = Self::default();

meilisearch/src/error.rs

@@ -23,8 +23,6 @@ pub enum MeilisearchHttpError {
     InvalidContentType(String, Vec<String>),
     #[error("Document `{0}` not found.")]
     DocumentNotFound(String),
-    #[error("Document `{0}` not found.")]
-    InvalidDocumentId(String),
     #[error("Sending an empty filter is forbidden.")]
     EmptyFilter,
     #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
@@ -61,10 +59,6 @@ pub enum MeilisearchHttpError {
     Join(#[from] JoinError),
     #[error("Invalid request: missing `hybrid` parameter when both `q` and `vector` are present.")]
     MissingSearchHybrid,
-    #[error("Invalid request: `prompt` parameter is required when `context` is present.")]
-    RecommendMissingPrompt,
-    #[error("Invalid request: one of the `prompt` or `id` parameters is required.")]
-    RecommendMissingPromptOrId,
 }
 
 impl ErrorCode for MeilisearchHttpError {
@@ -76,7 +70,6 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::MissingPayload(_) => Code::MissingPayload,
             MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,
             MeilisearchHttpError::DocumentNotFound(_) => Code::DocumentNotFound,
-            MeilisearchHttpError::InvalidDocumentId(_) => Code::InvalidDocumentId,
             MeilisearchHttpError::EmptyFilter => Code::InvalidDocumentFilter,
             MeilisearchHttpError::InvalidExpression(_, _) => Code::InvalidSearchFilter,
             MeilisearchHttpError::PayloadTooLarge(_) => Code::PayloadTooLarge,
@@ -93,8 +86,6 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
             MeilisearchHttpError::Join(_) => Code::Internal,
             MeilisearchHttpError::MissingSearchHybrid => Code::MissingSearchHybrid,
-            MeilisearchHttpError::RecommendMissingPrompt => Code::MissingPrompt,
-            MeilisearchHttpError::RecommendMissingPromptOrId => Code::MissingPromptOrId,
         }
     }
 }

meilisearch/src/middleware.rs

@@ -59,10 +59,12 @@ where
             let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
+                let request_pattern = req.match_pattern();
+                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
                 let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, request_path])
+                        .with_label_values(&[&request_method, metric_path])
                         .start_timer(),
                 );
             }

meilisearch/src/routes/indexes/facet_search.rs

@@ -14,9 +14,7 @@ use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, SearchQuery,
-    DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
+    add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, RankingScoreThreshold, SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET
 };
 use crate::search_queue::SearchQueue;
 
@@ -46,6 +44,8 @@ pub struct FacetSearchQuery {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
 }
 
 pub async fn search(
@@ -103,6 +103,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = value;
 
         SearchQuery {
@@ -128,6 +129,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             vector,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         }
     }
 }

meilisearch/src/routes/indexes/mod.rs

@@ -27,7 +27,6 @@ use crate::Opt;
 
 pub mod documents;
 pub mod facet_search;
-pub mod recommend;
 pub mod search;
 pub mod settings;
 
@@ -49,7 +48,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
             .service(web::scope("/documents").configure(documents::configure))
             .service(web::scope("/search").configure(search::configure))
             .service(web::scope("/facet-search").configure(facet_search::configure))
-            .service(web::scope("/recommend").configure(recommend::configure))
             .service(web::scope("/settings").configure(settings::configure)),
     );
 }

meilisearch/src/routes/indexes/recommend.rs

@@ -1,53 +0,0 @@
-use actix_web::web::{self, Data};
-use actix_web::{HttpRequest, HttpResponse};
-use deserr::actix_web::AwebJson;
-use index_scheduler::IndexScheduler;
-use meilisearch_types::deserr::DeserrJsonError;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::index_uid::IndexUid;
-use meilisearch_types::keys::actions;
-use tracing::debug;
-
-use super::ActionPolicy;
-use crate::analytics::Analytics;
-use crate::extractors::authentication::GuardedData;
-use crate::extractors::sequential_extractor::SeqHandler;
-use crate::search::{perform_recommend, RecommendQuery, SearchKind};
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(SeqHandler(recommend))));
-}
-
-pub async fn recommend(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
-    index_uid: web::Path<String>,
-    params: AwebJson<RecommendQuery, DeserrJsonError>,
-    _req: HttpRequest,
-    _analytics: web::Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-
-    // TODO analytics
-
-    let query = params.into_inner();
-    debug!(parameters = ?query, "Recommend post");
-
-    let index = index_scheduler.index(&index_uid)?;
-
-    let features = index_scheduler.features();
-
-    features.check_vector("Using the recommend API.")?;
-
-    let (embedder_name, embedder) =
-        SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
-
-    let recommendations = tokio::task::spawn_blocking(move || {
-        perform_recommend(&index, query, embedder_name, embedder)
-    })
-    .await?;
-
-    let recommendations = recommendations?;
-
-    debug!(returns = ?recommendations, "Recommend post");
-    Ok(HttpResponse::Ok().json(recommendations))
-}

meilisearch/src/routes/indexes/search.rs

@@ -19,9 +19,10 @@ use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
 use crate::search::{
-    add_search_rules, perform_search, HybridQuery, MatchingStrategy, SearchKind, SearchQuery,
-    SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
+    add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
+    SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
 };
 use crate::search_queue::SearchQueue;
 
@@ -82,6 +83,21 @@ pub struct SearchQueryGet {
     pub hybrid_embedder: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchSemanticRatio>)]
     pub hybrid_semantic_ratio: Option<SemanticRatioGet>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
+#[deserr(try_from(String) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
+pub struct RankingScoreThresholdGet(RankingScoreThreshold);
+
+impl std::convert::TryFrom<String> for RankingScoreThresholdGet {
+    type Error = InvalidSearchRankingScoreThreshold;
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        let f: f64 = s.parse().map_err(|_| InvalidSearchRankingScoreThreshold)?;
+        Ok(RankingScoreThresholdGet(RankingScoreThreshold::try_from(f)?))
+    }
 }
 
 #[derive(Debug, Clone, Copy, Default, PartialEq, deserr::Deserr)]
@@ -152,6 +168,7 @@ impl From<SearchQueryGet> for SearchQuery {
             matching_strategy: other.matching_strategy,
             attributes_to_search_on: other.attributes_to_search_on.map(|o| o.into_iter().collect()),
             hybrid,
+            ranking_score_threshold: other.ranking_score_threshold.map(|o| o.0),
         }
     }
 }

meilisearch/src/routes/mod.rs

@@ -376,12 +376,6 @@ async fn get_version(
     })
 }
 
-#[derive(Serialize)]
-struct KeysResponse {
-    private: Option<String>,
-    public: Option<String>,
-}
-
 pub async fn get_health(
     req: HttpRequest,
     index_scheduler: Data<IndexScheduler>,

meilisearch/src/search.rs

@@ -86,6 +86,26 @@ pub struct SearchQuery {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Deserr)]
+#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
+pub struct RankingScoreThreshold(f64);
+
+impl std::convert::TryFrom<f64> for RankingScoreThreshold {
+    type Error = InvalidSearchRankingScoreThreshold;
+
+    fn try_from(f: f64) -> Result<Self, Self::Error> {
+        // the suggested "fix" is: `!(0.0..=1.0).contains(&f)`` which is allegedly less readable
+        #[allow(clippy::manual_range_contains)]
+        if f > 1.0 || f < 0.0 {
+            Err(InvalidSearchRankingScoreThreshold)
+        } else {
+            Ok(RankingScoreThreshold(f))
+        }
+    }
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Deserr)]
@@ -126,7 +146,7 @@ impl SearchKind {
         Ok(Self::Hybrid { embedder_name, embedder, semantic_ratio })
     }
 
-    pub(crate) fn embedder(
+    fn embedder(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
         embedder_name: Option<&str>,
@@ -251,6 +271,8 @@ pub struct SearchQueryWithIndex {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
 }
 
 impl SearchQueryWithIndex {
@@ -279,6 +301,7 @@ impl SearchQueryWithIndex {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = self;
         (
             index_uid,
@@ -305,6 +328,7 @@ impl SearchQueryWithIndex {
                 matching_strategy,
                 attributes_to_search_on,
                 hybrid,
+                ranking_score_threshold,
                 // do not use ..Default::default() here,
                 // rather add any missing field from `SearchQuery` to `SearchQueryWithIndex`
             },
@@ -312,32 +336,6 @@ impl SearchQueryWithIndex {
     }
 }
 
-#[derive(Debug, Clone, Default, PartialEq, Deserr)]
-#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
-pub struct RecommendQuery {
-    #[deserr(default, error = DeserrJsonError<InvalidRecommendId>)]
-    pub id: Option<String>,
-    #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
-    pub offset: usize,
-    #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
-    pub limit: usize,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchFilter>)]
-    pub filter: Option<Value>,
-    #[deserr(default, error = DeserrJsonError<InvalidEmbedder>, default)]
-    pub embedder: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
-    pub attributes_to_retrieve: Option<BTreeSet<String>>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchShowRankingScore>, default)]
-    pub show_ranking_score: bool,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchShowRankingScoreDetails>, default)]
-    pub show_ranking_score_details: bool,
-
-    #[deserr(default, error = DeserrJsonError<InvalidRecommendPrompt>)]
-    pub prompt: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidRecommendContext>)]
-    pub context: Option<Value>,
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr)]
 #[deserr(rename_all = camelCase)]
 pub enum MatchingStrategy {
@@ -419,17 +417,6 @@ pub struct SearchResult {
     pub used_negative_operator: bool,
 }
 
-#[derive(Serialize, Debug, Clone, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub struct RecommendResult {
-    pub hits: Vec<SearchHit>,
-    pub id: Option<String>,
-    pub prompt: Option<String>,
-    pub processing_time_ms: u128,
-    #[serde(flatten)]
-    pub hits_info: HitsInfo,
-}
-
 #[derive(Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResultWithIndex {
@@ -490,6 +477,7 @@ fn prepare_search<'t>(
 ) -> Result<(milli::Search<'t>, bool, usize, usize), MeilisearchHttpError> {
     let mut search = index.search(rtxn);
     search.time_budget(time_budget);
+    search.ranking_score_threshold(query.ranking_score_threshold.map(|rst| rst.0));
 
     match search_kind {
         SearchKind::KeywordOnly => {
@@ -531,11 +519,16 @@ fn prepare_search<'t>(
         .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
 
     search.exhaustive_number_hits(is_finite_pagination);
-    search.scoring_strategy(if query.show_ranking_score || query.show_ranking_score_details {
-        ScoringStrategy::Detailed
-    } else {
-        ScoringStrategy::Skip
-    });
+    search.scoring_strategy(
+        if query.show_ranking_score
+            || query.show_ranking_score_details
+            || query.ranking_score_threshold.is_some()
+        {
+            ScoringStrategy::Detailed
+        } else {
+            ScoringStrategy::Skip
+        },
+    );
 
     // compute the offset on the limit depending on the pagination mode.
     let (offset, limit) = if is_finite_pagination {
@@ -833,153 +826,6 @@ pub fn perform_facet_search(
     })
 }
 
-pub fn perform_recommend(
-    index: &Index,
-    query: RecommendQuery,
-    embedder_name: String,
-    embedder: Arc<Embedder>,
-) -> Result<RecommendResult, MeilisearchHttpError> {
-    let before_search = Instant::now();
-    let rtxn = index.read_txn()?;
-
-    let internal_id = query
-        .id
-        .as_deref()
-        .map(|id| -> Result<_, MeilisearchHttpError> {
-            Ok(index
-                .external_documents_ids()
-                .get(&rtxn, id)?
-                .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(id.to_owned()))?)
-        })
-        .transpose()?;
-
-    let mut recommend = match (query.prompt.as_deref(), internal_id, query.context) {
-        (None, Some(internal_id), None) => milli::Recommend::with_docid(
-            internal_id,
-            query.offset,
-            query.limit,
-            index,
-            &rtxn,
-            embedder_name,
-            embedder,
-        ),
-        (Some(prompt), internal_id, context) => milli::Recommend::with_prompt(
-            prompt,
-            internal_id,
-            context,
-            query.offset,
-            query.limit,
-            index,
-            &rtxn,
-            embedder_name,
-            embedder,
-        ),
-        (None, _, Some(_)) => return Err(MeilisearchHttpError::RecommendMissingPrompt.into()),
-        (None, None, None) => return Err(MeilisearchHttpError::RecommendMissingPromptOrId.into()),
-    };
-
-    if let Some(ref filter) = query.filter {
-        if let Some(facets) = parse_filter(filter)? {
-            recommend.filter(facets);
-        }
-    }
-
-    let milli::SearchResult {
-        documents_ids,
-        matching_words: _,
-        candidates,
-        document_scores,
-        degraded: _,
-        used_negative_operator: _,
-    } = recommend.execute()?;
-
-    let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
-
-    let displayed_ids = index
-        .displayed_fields_ids(&rtxn)?
-        .map(|fields| fields.into_iter().collect::<BTreeSet<_>>())
-        .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
-
-    let fids = |attrs: &BTreeSet<String>| {
-        let mut ids = BTreeSet::new();
-        for attr in attrs {
-            if attr == "*" {
-                ids = displayed_ids.clone();
-                break;
-            }
-
-            if let Some(id) = fields_ids_map.id(attr) {
-                ids.insert(id);
-            }
-        }
-        ids
-    };
-
-    // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default),
-    // but these attributes must be also be present
-    // - in the fields_ids_map
-    // - in the displayed attributes
-    let to_retrieve_ids: BTreeSet<_> = query
-        .attributes_to_retrieve
-        .as_ref()
-        .map(fids)
-        .unwrap_or_else(|| displayed_ids.clone())
-        .intersection(&displayed_ids)
-        .cloned()
-        .collect();
-
-    let mut documents = Vec::new();
-    let documents_iter = index.documents(&rtxn, documents_ids)?;
-
-    for ((_id, obkv), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
-        // First generate a document with all the displayed fields
-        let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
-
-        // select the attributes to retrieve
-        let attributes_to_retrieve = to_retrieve_ids
-            .iter()
-            .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
-        let document =
-            permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
-
-        let ranking_score =
-            query.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
-        let ranking_score_details =
-            query.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
-
-        let hit = SearchHit {
-            document,
-            formatted: Default::default(),
-            matches_position: None,
-            ranking_score_details,
-            ranking_score,
-        };
-        documents.push(hit);
-    }
-
-    let max_total_hits = index
-        .pagination_max_total_hits(&rtxn)
-        .map_err(milli::Error::from)?
-        .map(|x| x as usize)
-        .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
-
-    let number_of_hits = min(candidates.len() as usize, max_total_hits);
-    let hits_info = HitsInfo::OffsetLimit {
-        limit: query.limit,
-        offset: query.offset,
-        estimated_total_hits: number_of_hits,
-    };
-
-    let result = RecommendResult {
-        hits: documents,
-        hits_info,
-        id: query.id,
-        prompt: query.prompt,
-        processing_time_ms: before_search.elapsed().as_millis(),
-    };
-    Ok(result)
-}
-
 fn insert_geo_distance(sorts: &[String], document: &mut Document) {
     lazy_static::lazy_static! {
         static ref GEO_REGEX: Regex =

meilitool/src/main.rs

@@ -129,7 +129,7 @@ fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> {
         }
     }
 
-    eprintln!("Sucessfully deleted {count} content files from disk!");
+    eprintln!("Successfully deleted {count} content files from disk!");
 
     Ok(())
 }

milli/Cargo.toml

@@ -26,7 +26,7 @@ flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"
 fxhash = "0.2.1"
 geoutils = "0.5.1"
-grenad = { version = "0.4.5", default-features = false, features = [
+grenad = { version = "0.4.6", default-features = false, features = [
     "rayon",
     "tempfile",
 ] }

milli/examples/search.rs

@@ -49,7 +49,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             let start = Instant::now();
 
             let mut ctx = SearchContext::new(&index, &txn);
-            let universe = filtered_universe(ctx.index, ctx.txn, &None)?;
+            let universe = filtered_universe(&ctx, &None)?;
 
             let docs = execute_search(
                 &mut ctx,
@@ -66,6 +66,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 &mut DefaultSearchLogger,
                 logger,
                 TimeBudget::max(),
+                None,
             )?;
             if let Some((logger, dir)) = detailed_logger {
                 logger.finish(&mut ctx, Path::new(dir))?;

milli/src/documents/builder.rs

@@ -203,7 +203,7 @@ fn parse_csv_header(header: &str) -> (&str, AllowedType) {
             "string" => (field_name, AllowedType::String),
             "boolean" => (field_name, AllowedType::Boolean),
             "number" => (field_name, AllowedType::Number),
-            // if the pattern isn't reconized, we keep the whole field.
+            // if the pattern isn't recognized, we keep the whole field.
             _otherwise => (header, AllowedType::String),
         },
         None => (header, AllowedType::String),

milli/src/lib.rs

@@ -59,7 +59,6 @@ pub use self::heed_codec::{
 };
 pub use self::index::Index;
 pub use self::search::facet::{FacetValueHit, SearchForFacetValues};
-pub use self::search::recommend::Recommend;
 pub use self::search::{
     FacetDistribution, Filter, FormatOptions, MatchBounds, MatcherBuilder, MatchingWords, OrderBy,
     Search, SearchResult, SemanticSearch, TermsMatchingStrategy, DEFAULT_VALUES_PER_FACET,

milli/src/prompt/document.rs

@@ -29,7 +29,7 @@ impl ParsedValue {
 }
 
 impl<'a> Document<'a> {
-    pub fn from_deladd_obkv(
+    pub fn new(
         data: obkv::KvReaderU16<'a>,
         side: DelAdd,
         inverted_field_map: &'a FieldsIdsMap,
@@ -48,20 +48,6 @@ impl<'a> Document<'a> {
         Self(out_data)
     }
 
-    pub fn from_doc_obkv(
-        data: obkv::KvReaderU16<'a>,
-        inverted_field_map: &'a FieldsIdsMap,
-    ) -> Self {
-        let mut out_data = BTreeMap::new();
-        for (fid, raw) in data {
-            let Some(name) = inverted_field_map.name(fid) else {
-                continue;
-            };
-            out_data.insert(name, (raw, ParsedValue::empty()));
-        }
-        Self(out_data)
-    }
-
     fn is_empty(&self) -> bool {
         self.0.is_empty()
     }

milli/src/prompt/mod.rs

@@ -2,7 +2,6 @@ mod context;
 mod document;
 pub(crate) mod error;
 mod fields;
-pub mod recommend;
 mod template_checker;
 
 use std::convert::TryFrom;
@@ -10,7 +9,7 @@ use std::convert::TryFrom;
 use error::{NewPromptError, RenderPromptError};
 
 use self::context::Context;
-pub use self::document::Document;
+use self::document::Document;
 use crate::update::del_add::DelAdd;
 use crate::FieldsIdsMap;
 
@@ -96,7 +95,7 @@ impl Prompt {
         side: DelAdd,
         field_id_map: &FieldsIdsMap,
     ) -> Result<String, RenderPromptError> {
-        let document = Document::from_deladd_obkv(document, side, field_id_map);
+        let document = Document::new(document, side, field_id_map);
         let context = Context::new(&document, field_id_map);
 
         self.template.render(&context).map_err(RenderPromptError::missing_context)

milli/src/prompt/recommend.rs

@@ -1,112 +0,0 @@
-use liquid::model::{
-    DisplayCow, KStringCow, ObjectRender, ObjectSource, State, Value as LiquidValue,
-};
-use liquid::{ObjectView, ValueView};
-
-use super::document::Document;
-
-#[derive(Clone, Debug)]
-pub struct Context<'a> {
-    document: Option<&'a Document<'a>>,
-    context: Option<liquid::Object>,
-}
-
-impl<'a> Context<'a> {
-    pub fn new(document: Option<&'a Document<'a>>, context: Option<serde_json::Value>) -> Self {
-        /// FIXME: unwrap
-        let context = context.map(|context| liquid::to_object(&context).unwrap());
-        Self { document, context }
-    }
-}
-
-impl<'a> ObjectView for Context<'a> {
-    fn as_value(&self) -> &dyn ValueView {
-        self
-    }
-
-    fn size(&self) -> i64 {
-        match (self.context.as_ref(), self.document.as_ref()) {
-            (None, None) => 0,
-            (None, Some(_)) => 1,
-            (Some(_), None) => 1,
-            (Some(_), Some(_)) => 2,
-        }
-    }
-
-    fn keys<'k>(&'k self) -> Box<dyn Iterator<Item = KStringCow<'k>> + 'k> {
-        let keys = match (self.context.as_ref(), self.document.as_ref()) {
-            (None, None) => [].as_slice(),
-            (None, Some(_)) => ["doc"].as_slice(),
-            (Some(_), None) => ["context"].as_slice(),
-            (Some(_), Some(_)) => ["context", "doc"].as_slice(),
-        };
-
-        Box::new(keys.iter().map(|s| KStringCow::from_static(s)))
-    }
-
-    fn values<'k>(&'k self) -> Box<dyn Iterator<Item = &'k dyn ValueView> + 'k> {
-        Box::new(
-            self.context
-                .as_ref()
-                .map(|context| context.as_value())
-                .into_iter()
-                .chain(self.document.map(|document| document.as_value()).into_iter()),
-        )
-    }
-
-    fn iter<'k>(&'k self) -> Box<dyn Iterator<Item = (KStringCow<'k>, &'k dyn ValueView)> + 'k> {
-        Box::new(self.keys().zip(self.values()))
-    }
-
-    fn contains_key(&self, index: &str) -> bool {
-        index == "context" || index == "doc"
-    }
-
-    fn get<'s>(&'s self, index: &str) -> Option<&'s dyn ValueView> {
-        match index {
-            "context" => self.context.as_ref().map(|context| context.as_value()),
-            "doc" => self.document.as_ref().map(|doc| doc.as_value()),
-            _ => None,
-        }
-    }
-}
-
-impl<'a> ValueView for Context<'a> {
-    fn as_debug(&self) -> &dyn std::fmt::Debug {
-        self
-    }
-
-    fn render(&self) -> liquid::model::DisplayCow<'_> {
-        DisplayCow::Owned(Box::new(ObjectRender::new(self)))
-    }
-
-    fn source(&self) -> liquid::model::DisplayCow<'_> {
-        DisplayCow::Owned(Box::new(ObjectSource::new(self)))
-    }
-
-    fn type_name(&self) -> &'static str {
-        "object"
-    }
-
-    fn query_state(&self, state: liquid::model::State) -> bool {
-        match state {
-            State::Truthy => true,
-            State::DefaultValue | State::Empty | State::Blank => false,
-        }
-    }
-
-    fn to_kstr(&self) -> liquid::model::KStringCow<'_> {
-        let s = ObjectRender::new(self).to_string();
-        KStringCow::from_string(s)
-    }
-
-    fn to_value(&self) -> LiquidValue {
-        LiquidValue::Object(
-            self.iter().map(|(k, x)| (k.to_string().into(), x.to_value())).collect(),
-        )
-    }
-
-    fn as_object(&self) -> Option<&dyn ObjectView> {
-        Some(self)
-    }
-}

milli/src/search/hybrid.rs

@@ -169,6 +169,7 @@ impl<'a> Search<'a> {
             index: self.index,
             semantic: self.semantic.clone(),
             time_budget: self.time_budget.clone(),
+            ranking_score_threshold: self.ranking_score_threshold,
         };
 
         let semantic = search.semantic.take();

milli/src/search/mod.rs

@@ -24,7 +24,6 @@ pub mod facet;
 mod fst_utils;
 pub mod hybrid;
 pub mod new;
-pub mod recommend;
 
 #[derive(Debug, Clone)]
 pub struct SemanticSearch {
@@ -50,6 +49,7 @@ pub struct Search<'a> {
     index: &'a Index,
     semantic: Option<SemanticSearch>,
     time_budget: TimeBudget,
+    ranking_score_threshold: Option<f64>,
 }
 
 impl<'a> Search<'a> {
@@ -70,6 +70,7 @@ impl<'a> Search<'a> {
             index,
             semantic: None,
             time_budget: TimeBudget::max(),
+            ranking_score_threshold: None,
         }
     }
 
@@ -146,10 +147,18 @@ impl<'a> Search<'a> {
         self
     }
 
+    pub fn ranking_score_threshold(
+        &mut self,
+        ranking_score_threshold: Option<f64>,
+    ) -> &mut Search<'a> {
+        self.ranking_score_threshold = ranking_score_threshold;
+        self
+    }
+
     pub fn execute_for_candidates(&self, has_vector_search: bool) -> Result<RoaringBitmap> {
         if has_vector_search {
             let ctx = SearchContext::new(self.index, self.rtxn);
-            filtered_universe(ctx.index, ctx.txn, &self.filter)
+            filtered_universe(&ctx, &self.filter)
         } else {
             Ok(self.execute()?.candidates)
         }
@@ -162,7 +171,7 @@ impl<'a> Search<'a> {
             ctx.searchable_attributes(searchable_attributes)?;
         }
 
-        let universe = filtered_universe(ctx.index, ctx.txn, &self.filter)?;
+        let universe = filtered_universe(&ctx, &self.filter)?;
         let PartialSearchResult {
             located_query_terms,
             candidates,
@@ -184,6 +193,7 @@ impl<'a> Search<'a> {
                     embedder_name,
                     embedder,
                     self.time_budget.clone(),
+                    self.ranking_score_threshold,
                 )?
             }
             _ => execute_search(
@@ -201,6 +211,7 @@ impl<'a> Search<'a> {
                 &mut DefaultSearchLogger,
                 &mut DefaultSearchLogger,
                 self.time_budget.clone(),
+                self.ranking_score_threshold,
             )?,
         };
 
@@ -239,6 +250,7 @@ impl fmt::Debug for Search<'_> {
             index: _,
             semantic,
             time_budget,
+            ranking_score_threshold,
         } = self;
         f.debug_struct("Search")
             .field("query", query)
@@ -257,6 +269,7 @@ impl fmt::Debug for Search<'_> {
                 &semantic.as_ref().map(|semantic| &semantic.embedder_name),
             )
             .field("time_budget", time_budget)
+            .field("ranking_score_threshold", ranking_score_threshold)
             .finish()
     }
 }

milli/src/search/new/bucket_sort.rs

@@ -28,6 +28,7 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
     scoring_strategy: ScoringStrategy,
     logger: &mut dyn SearchLogger<Q>,
     time_budget: TimeBudget,
+    ranking_score_threshold: Option<f64>,
 ) -> Result<BucketSortOutput> {
     logger.initial_query(query);
     logger.ranking_rules(&ranking_rules);
@@ -144,6 +145,7 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
                 ctx,
                 from,
                 length,
+                ranking_score_threshold,
                 logger,
                 &mut valid_docids,
                 &mut valid_scores,
@@ -164,7 +166,9 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
             loop {
                 let bucket = std::mem::take(&mut ranking_rule_universes[cur_ranking_rule_index]);
                 ranking_rule_scores.push(ScoreDetails::Skipped);
+
                 maybe_add_to_results!(bucket);
+
                 ranking_rule_scores.pop();
 
                 if cur_ranking_rule_index == 0 {
@@ -220,6 +224,17 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
         debug_assert!(
             ranking_rule_universes[cur_ranking_rule_index].is_superset(&next_bucket.candidates)
         );
+
+        if let Some(ranking_score_threshold) = ranking_score_threshold {
+            let current_score = ScoreDetails::global_score(ranking_rule_scores.iter());
+            if current_score < ranking_score_threshold {
+                all_candidates -=
+                    next_bucket.candidates | &ranking_rule_universes[cur_ranking_rule_index];
+                back!();
+                continue;
+            }
+        }
+
         ranking_rule_universes[cur_ranking_rule_index] -= &next_bucket.candidates;
 
         if cur_ranking_rule_index == ranking_rules_len - 1
@@ -262,6 +277,7 @@ fn maybe_add_to_results<'ctx, Q: RankingRuleQueryTrait>(
     ctx: &mut SearchContext<'ctx>,
     from: usize,
     length: usize,
+    ranking_score_threshold: Option<f64>,
     logger: &mut dyn SearchLogger<Q>,
 
     valid_docids: &mut Vec<u32>,
@@ -279,6 +295,15 @@ fn maybe_add_to_results<'ctx, Q: RankingRuleQueryTrait>(
     ranking_rule_scores: &[ScoreDetails],
     candidates: RoaringBitmap,
 ) -> Result<()> {
+    // remove candidates from the universe without adding them to result if their score is below the threshold
+    if let Some(ranking_score_threshold) = ranking_score_threshold {
+        let score = ScoreDetails::global_score(ranking_rule_scores.iter());
+        if score < ranking_score_threshold {
+            *all_candidates -= candidates | &ranking_rule_universes[cur_ranking_rule_index];
+            return Ok(());
+        }
+    }
+
     // First apply the distinct rule on the candidates, reducing the universes if necessary
     let candidates = if let Some(distinct_fid) = distinct_fid {
         let DistinctOutput { remaining, excluded } =

milli/src/search/new/geo_sort.rs

@@ -42,7 +42,7 @@ fn facet_number_values<'a>(
 }
 
 /// Define the strategy used by the geo sort.
-/// The paramater represents the cache size, and, in the case of the Dynamic strategy,
+/// The parameter represents the cache size, and, in the case of the Dynamic strategy,
 /// the point where we move from using the iterative strategy to the rtree.
 #[derive(Debug, Clone, Copy)]
 pub enum Strategy {

milli/src/search/new/matches/mod.rs

@@ -134,7 +134,7 @@ impl<'t> Matcher<'t, '_> {
             for (token_position, word_position, word) in words_positions {
                 partial = match partial.match_token(word) {
                     // token matches the partial match, but the match is not full,
-                    // we temporarly save the current token then we try to match the next one.
+                    // we temporarily save the current token then we try to match the next one.
                     Some(MatchType::Partial(partial)) => {
                         potential_matches.push((token_position, word_position, partial.char_len()));
                         partial
@@ -507,7 +507,7 @@ mod tests {
     impl<'a> MatcherBuilder<'a> {
         fn new_test(rtxn: &'a heed::RoTxn, index: &'a TempIndex, query: &str) -> Self {
             let mut ctx = SearchContext::new(index, rtxn);
-            let universe = filtered_universe(ctx.index, ctx.txn, &None).unwrap();
+            let universe = filtered_universe(&ctx, &None).unwrap();
             let crate::search::PartialSearchResult { located_query_terms, .. } = execute_search(
                 &mut ctx,
                 Some(query),
@@ -523,6 +523,7 @@ mod tests {
                 &mut crate::DefaultSearchLogger,
                 &mut crate::DefaultSearchLogger,
                 TimeBudget::max(),
+                None,
             )
             .unwrap();
 
@@ -722,7 +723,7 @@ mod tests {
             @"…void void void void void split the world void void"
         );
 
-        // Text containing matches with diferent density.
+        // Text containing matches with different density.
         let text = "split void the void void world void void void void void void void void void void split the world void void";
         let mut matcher = builder.build(text);
         // crop should return 10 last words with a marker at the start.

milli/src/search/new/mod.rs

@@ -530,15 +530,11 @@ fn resolve_sort_criteria<'ctx, Query: RankingRuleQueryTrait>(
     Ok(())
 }
 
-pub fn filtered_universe(
-    index: &Index,
-    txn: &RoTxn<'_>,
-    filters: &Option<Filter>,
-) -> Result<RoaringBitmap> {
+pub fn filtered_universe(ctx: &SearchContext, filters: &Option<Filter>) -> Result<RoaringBitmap> {
     Ok(if let Some(filters) = filters {
-        filters.evaluate(txn, index)?
+        filters.evaluate(ctx.txn, ctx.index)?
     } else {
-        index.documents_ids(txn)?
+        ctx.index.documents_ids(ctx.txn)?
     })
 }
 
@@ -555,6 +551,7 @@ pub fn execute_vector_search(
     embedder_name: &str,
     embedder: &Embedder,
     time_budget: TimeBudget,
+    ranking_score_threshold: Option<f64>,
 ) -> Result<PartialSearchResult> {
     check_sort_criteria(ctx, sort_criteria.as_ref())?;
 
@@ -584,6 +581,7 @@ pub fn execute_vector_search(
         scoring_strategy,
         placeholder_search_logger,
         time_budget,
+        ranking_score_threshold,
     )?;
 
     Ok(PartialSearchResult {
@@ -613,6 +611,7 @@ pub fn execute_search(
     placeholder_search_logger: &mut dyn SearchLogger<PlaceholderQuery>,
     query_graph_logger: &mut dyn SearchLogger<QueryGraph>,
     time_budget: TimeBudget,
+    ranking_score_threshold: Option<f64>,
 ) -> Result<PartialSearchResult> {
     check_sort_criteria(ctx, sort_criteria.as_ref())?;
 
@@ -701,6 +700,7 @@ pub fn execute_search(
             scoring_strategy,
             query_graph_logger,
             time_budget,
+            ranking_score_threshold,
         )?
     } else {
         let ranking_rules =
@@ -715,6 +715,7 @@ pub fn execute_search(
             scoring_strategy,
             placeholder_search_logger,
             time_budget,
+            ranking_score_threshold,
         )?
     };
 

milli/src/search/new/query_term/parse_query.rs

@@ -119,7 +119,7 @@ pub fn located_query_terms_from_tokens(
                             if let Some(located_query_term) = phrase.build(ctx) {
                                 // as we are evaluating a negative operator we put the phrase
                                 // in the negative one *but* we don't reset the negative operator
-                                // as we are immediatly starting a new negative phrase.
+                                // as we are immediately starting a new negative phrase.
                                 if negative_phrase {
                                     negative_phrases.push(located_query_term);
                                 } else {

milli/src/search/recommend.rs

@@ -1,205 +0,0 @@
-use std::sync::Arc;
-
-use ordered_float::OrderedFloat;
-use roaring::RoaringBitmap;
-use serde_json::Value;
-
-use crate::score_details::{self, ScoreDetails};
-use crate::vector::Embedder;
-use crate::{filtered_universe, DocumentId, Filter, Index, Result, SearchResult};
-
-enum RecommendKind<'a> {
-    Id(DocumentId),
-    Prompt { prompt: &'a str, context: Option<Value>, id: Option<DocumentId> },
-}
-
-pub struct Recommend<'a> {
-    kind: RecommendKind<'a>,
-    // this should be linked to the String in the query
-    filter: Option<Filter<'a>>,
-    offset: usize,
-    limit: usize,
-    rtxn: &'a heed::RoTxn<'a>,
-    index: &'a Index,
-    embedder_name: String,
-    embedder: Arc<Embedder>,
-}
-
-impl<'a> Recommend<'a> {
-    pub fn with_docid(
-        id: DocumentId,
-        offset: usize,
-        limit: usize,
-        index: &'a Index,
-        rtxn: &'a heed::RoTxn<'a>,
-        embedder_name: String,
-        embedder: Arc<Embedder>,
-    ) -> Self {
-        Self {
-            kind: RecommendKind::Id(id),
-            filter: None,
-            offset,
-            limit,
-            rtxn,
-            index,
-            embedder_name,
-            embedder,
-        }
-    }
-
-    pub fn with_prompt(
-        prompt: &'a str,
-        id: Option<DocumentId>,
-        context: Option<Value>,
-        offset: usize,
-        limit: usize,
-        index: &'a Index,
-        rtxn: &'a heed::RoTxn<'a>,
-        embedder_name: String,
-        embedder: Arc<Embedder>,
-    ) -> Self {
-        Self {
-            kind: RecommendKind::Prompt { prompt, context, id },
-            filter: None,
-            offset,
-            limit,
-            rtxn,
-            index,
-            embedder_name,
-            embedder,
-        }
-    }
-
-    pub fn filter(&mut self, filter: Filter<'a>) -> &mut Self {
-        self.filter = Some(filter);
-        self
-    }
-
-    pub fn execute(&self) -> Result<SearchResult> {
-        let universe = filtered_universe(self.index, self.rtxn, &self.filter)?;
-
-        let embedder_index =
-            self.index
-                .embedder_category_id
-                .get(self.rtxn, &self.embedder_name)?
-                .ok_or_else(|| crate::UserError::InvalidEmbedder(self.embedder_name.to_owned()))?;
-
-        let writer_index = (embedder_index as u16) << 8;
-        let readers: std::result::Result<Vec<_>, _> = (0..=u8::MAX)
-            .map_while(|k| {
-                arroy::Reader::open(self.rtxn, writer_index | (k as u16), self.index.vector_arroy)
-                    .map(Some)
-                    .or_else(|e| match e {
-                        arroy::Error::MissingMetadata => Ok(None),
-                        e => Err(e),
-                    })
-                    .transpose()
-            })
-            .collect();
-
-        let readers = readers?;
-
-        let mut results = Vec::new();
-
-        /// FIXME: make id optional...
-        let id = match &self.kind {
-            RecommendKind::Id(id) => *id,
-            RecommendKind::Prompt { prompt, context, id } => id.unwrap(),
-        };
-
-        let personalization_vector = if let RecommendKind::Prompt { prompt, context, id } =
-            &self.kind
-        {
-            let fields_ids_map = self.index.fields_ids_map(self.rtxn)?;
-
-            let document = if let Some(id) = id {
-                Some(self.index.iter_documents(self.rtxn, std::iter::once(*id))?.next().unwrap()?.1)
-            } else {
-                None
-            };
-            let document = document
-                .map(|document| crate::prompt::Document::from_doc_obkv(document, &fields_ids_map));
-
-            let context =
-                crate::prompt::recommend::Context::new(document.as_ref(), context.clone());
-
-            /// FIXME: handle error bad template
-            let template =
-                liquid::ParserBuilder::new().stdlib().build().unwrap().parse(prompt).unwrap();
-
-            /// FIXME: handle error bad context
-            let rendered = template.render(&context).unwrap();
-
-            /// FIXME: handle embedding error
-            Some(self.embedder.embed_one(rendered).unwrap())
-        } else {
-            None
-        };
-
-        for reader in readers.iter() {
-            let nns_by_item = reader.nns_by_item(
-                self.rtxn,
-                id,
-                self.limit + self.offset + 1,
-                None,
-                Some(&universe),
-            )?;
-
-            if let Some(nns_by_item) = nns_by_item {
-                let mut nns = match &personalization_vector {
-                    Some(vector) => {
-                        let candidates: RoaringBitmap =
-                            nns_by_item.iter().map(|(docid, _)| docid).collect();
-                        reader.nns_by_vector(
-                            self.rtxn,
-                            vector,
-                            self.limit + self.offset + 1,
-                            None,
-                            Some(&candidates),
-                        )?
-                    }
-                    None => nns_by_item,
-                };
-
-                results.append(&mut nns);
-            }
-        }
-
-        results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance));
-
-        let mut documents_ids = Vec::with_capacity(self.limit);
-        let mut document_scores = Vec::with_capacity(self.limit);
-
-        // skip offset +1 to skip the target document that is normally returned
-        for (docid, distance) in results.into_iter().skip(self.offset) {
-            if documents_ids.len() == self.limit {
-                break;
-            }
-            if id == docid {
-                continue;
-            }
-
-            documents_ids.push(docid);
-
-            let score = 1.0 - distance;
-            let score = self
-                .embedder
-                .distribution()
-                .map(|distribution| distribution.shift(score))
-                .unwrap_or(score);
-
-            let score = ScoreDetails::Vector(score_details::Vector { similarity: Some(score) });
-
-            document_scores.push(vec![score]);
-        }
-
-        Ok(SearchResult {
-            matching_words: Default::default(),
-            candidates: universe,
-            documents_ids,
-            document_scores,
-            degraded: false,
-            used_negative_operator: false,
-        })
-    }
-}

milli/src/update/facet/incremental.rs

@@ -499,7 +499,7 @@ impl FacetsUpdateIncrementalInner {
                     ModificationResult::Expand | ModificationResult::Reduce { .. }
                 )
             {
-                // if any modification occured, insert it in the database.
+                // if any modification occurred, insert it in the database.
                 self.db.put(txn, &insertion_key.as_ref(), &updated_value)?;
                 Ok(insertion_key_modification)
             } else {

milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs

@@ -36,7 +36,7 @@ pub struct ExtractedFacetValues {
 
 /// Extracts the facet values of each faceted field of each document.
 ///
-/// Returns the generated grenad reader containing the docid the fid and the orginal value as key
+/// Returns the generated grenad reader containing the docid the fid and the original value as key
 /// and the normalized value as value extracted from the given chunk of documents.
 /// We need the fid of the geofields to correctly parse them as numbers if they were sent as strings initially.
 #[tracing::instrument(level = "trace", skip_all, target = "indexing::extract")]