update milli

Cargo.lock

@@ -1124,7 +1124,7 @@ dependencies = [
 [[package]]
 name = "filter-parser"
 version = "0.32.0"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.32.0#e1bc610d2722a8010216c45d5a32cbe3db18468e"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#dc179f6df67161689562cedbce41439a120e5fd4"
 dependencies = [
  "nom",
  "nom_locate",
@@ -1149,7 +1149,7 @@ dependencies = [
 [[package]]
 name = "flatten-serde-json"
 version = "0.32.0"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.32.0#e1bc610d2722a8010216c45d5a32cbe3db18468e"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#dc179f6df67161689562cedbce41439a120e5fd4"
 dependencies = [
  "serde_json",
 ]
@@ -1662,7 +1662,7 @@ dependencies = [
 [[package]]
 name = "json-depth-checker"
 version = "0.32.0"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.32.0#e1bc610d2722a8010216c45d5a32cbe3db18468e"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#dc179f6df67161689562cedbce41439a120e5fd4"
 dependencies = [
  "serde_json",
 ]
@@ -2190,7 +2190,7 @@ dependencies = [
 [[package]]
 name = "milli"
 version = "0.32.0"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.32.0#e1bc610d2722a8010216c45d5a32cbe3db18468e"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#dc179f6df67161689562cedbce41439a120e5fd4"
 dependencies = [
  "bimap",
  "bincode",

meilisearch-auth/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 enum-iterator = "0.7.0"
 hmac = "0.12.1"
 meilisearch-types = { path = "../meilisearch-types" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.32.0" }
+milli = { git = "https://github.com/meilisearch/milli.git", branch = "ease-search-results-pagination" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { version = "1.0.79", features = ["preserve_order"] }

meilisearch-http/src/analytics/segment_analytics.rs

@@ -10,7 +10,7 @@ use http::header::CONTENT_TYPE;
 use meilisearch_auth::SearchRules;
 use meilisearch_lib::index::{
     SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
 };
 use meilisearch_lib::index_controller::Stats;
 use meilisearch_lib::MeiliSearch;
@@ -270,8 +270,8 @@ impl Segment {
     }
 
     async fn run(mut self, meilisearch: MeiliSearch) {
-        const INTERVAL: Duration = Duration::from_secs(60 * 60); // one hour
-                                                                 // The first batch must be sent after one hour.
+        const INTERVAL: Duration = Duration::from_secs(60); // one minute
+                                                            // The first batch must be sent after one minute.
         let mut interval =
             tokio::time::interval_at(tokio::time::Instant::now() + INTERVAL, INTERVAL);
 
@@ -301,7 +301,7 @@ impl Segment {
                 .push(Identify {
                     context: Some(json!({
                         "app": {
-                            "version": env!("CARGO_PKG_VERSION").to_string(),
+                            "version": "prototype-pagination-2".to_string(),
                         },
                     })),
                     user: self.user.clone(),
@@ -369,6 +369,7 @@ pub struct SearchAggregator {
     // pagination
     max_limit: usize,
     max_offset: usize,
+    finite_pagination: usize,
 
     // formatting
     highlight_pre_tag: bool,
@@ -423,8 +424,15 @@ impl SearchAggregator {
             ret.max_terms_number = q.split_whitespace().count();
         }
 
-        ret.max_limit = query.limit;
-        ret.max_offset = query.offset.unwrap_or_default();
+        if query.limit.is_none() && query.offset.is_none() {
+            ret.max_limit = query.hits_per_page;
+            ret.max_offset = query.page.saturating_sub(1) * query.hits_per_page;
+            ret.finite_pagination = 1;
+        } else {
+            ret.max_limit = query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
+            ret.max_offset = query.offset.unwrap_or_default();
+            ret.finite_pagination = 0;
+        }
 
         ret.highlight_pre_tag = query.highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG();
         ret.highlight_post_tag = query.highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG();
@@ -479,6 +487,7 @@ impl SearchAggregator {
         // pagination
         self.max_limit = self.max_limit.max(other.max_limit);
         self.max_offset = self.max_offset.max(other.max_offset);
+        self.finite_pagination += other.finite_pagination;
 
         self.highlight_pre_tag |= other.highlight_pre_tag;
         self.highlight_post_tag |= other.highlight_post_tag;
@@ -521,6 +530,7 @@ impl SearchAggregator {
                 "pagination": {
                    "max_limit": self.max_limit,
                    "max_offset": self.max_offset,
+                   "finite_pagination": self.finite_pagination > self.total_received / 2,
                 },
                 "formatting": {
                     "highlight_pre_tag": self.highlight_pre_tag,

meilisearch-http/src/routes/indexes/search.rs

@@ -3,7 +3,7 @@ use log::debug;
 use meilisearch_auth::IndexSearchRules;
 use meilisearch_lib::index::{
     SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_HIT_PER_PAGE, DEFAULT_PAGE,
 };
 use meilisearch_lib::MeiliSearch;
 use meilisearch_types::error::ResponseError;
@@ -29,6 +29,10 @@ pub struct SearchQueryGet {
     q: Option<String>,
     offset: Option<usize>,
     limit: Option<usize>,
+    #[serde(default = "DEFAULT_PAGE")]
+    page: usize,
+    #[serde(default = "DEFAULT_HIT_PER_PAGE")]
+    hits_per_page: usize,
     attributes_to_retrieve: Option<CS<String>>,
     attributes_to_crop: Option<CS<String>>,
     #[serde(default = "DEFAULT_CROP_LENGTH")]
@@ -60,7 +64,9 @@ impl From<SearchQueryGet> for SearchQuery {
         Self {
             q: other.q,
             offset: other.offset,
-            limit: other.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+            limit: other.limit,
+            page: other.page,
+            hits_per_page: other.hits_per_page,
             attributes_to_retrieve: other
                 .attributes_to_retrieve
                 .map(|o| o.into_iter().collect()),

meilisearch-http/tests/search/errors.rs

@@ -74,7 +74,7 @@ async fn filter_invalid_syntax_object() {
     index.wait_task(1).await;
 
     let expected_response = json!({
-        "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `TO` or `_geoRadius` at `title & Glass`.\n1:14 title & Glass",
+        "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `TO`, `EXISTS`, `NOT EXISTS`, or `_geoRadius` at `title & Glass`.\n1:14 title & Glass",
         "code": "invalid_filter",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_filter"
@@ -101,7 +101,7 @@ async fn filter_invalid_syntax_array() {
     index.wait_task(1).await;
 
     let expected_response = json!({
-        "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `TO` or `_geoRadius` at `title & Glass`.\n1:14 title & Glass",
+        "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `TO`, `EXISTS`, `NOT EXISTS`, or `_geoRadius` at `title & Glass`.\n1:14 title & Glass",
         "code": "invalid_filter",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_filter"

meilisearch-lib/Cargo.toml

@@ -28,7 +28,7 @@ lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.32.0" }
+milli = { git = "https://github.com/meilisearch/milli.git", branch = "ease-search-results-pagination" }
 mime = "0.3.16"
 num_cpus = "1.13.1"
 obkv = "0.2.0"

meilisearch-lib/src/index/mod.rs

@@ -1,6 +1,7 @@
 pub use search::{
-    SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    HitsInfo, SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_HIT_PER_PAGE, DEFAULT_PAGE,
+    DEFAULT_SEARCH_LIMIT,
 };
 pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
 

meilisearch-lib/src/index/search.rs

@@ -26,6 +26,8 @@ pub const DEFAULT_CROP_LENGTH: fn() -> usize = || 10;
 pub const DEFAULT_CROP_MARKER: fn() -> String = || "…".to_string();
 pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "<em>".to_string();
 pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
+pub const DEFAULT_PAGE: fn() -> usize = || 1;
+pub const DEFAULT_HIT_PER_PAGE: fn() -> usize = || 20;
 
 /// The maximimum number of results that the engine
 /// will be able to return in one search call.
@@ -36,8 +38,11 @@ pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000;
 pub struct SearchQuery {
     pub q: Option<String>,
     pub offset: Option<usize>,
-    #[serde(default = "DEFAULT_SEARCH_LIMIT")]
-    pub limit: usize,
+    pub limit: Option<usize>,
+    #[serde(default = "DEFAULT_PAGE")]
+    pub page: usize,
+    #[serde(default = "DEFAULT_HIT_PER_PAGE")]
+    pub hits_per_page: usize,
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
     pub attributes_to_crop: Option<Vec<String>>,
     #[serde(default = "DEFAULT_CROP_LENGTH")]
@@ -71,15 +76,32 @@ pub struct SearchHit {
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
-    pub estimated_total_hits: u64,
     pub query: String,
-    pub limit: usize,
-    pub offset: usize,
     pub processing_time_ms: u128,
+    #[serde(flatten)]
+    pub hits_info: HitsInfo,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub facet_distribution: Option<BTreeMap<String, BTreeMap<String, u64>>>,
 }
 
+#[derive(Serialize, Debug, Clone, PartialEq)]
+#[serde(untagged)]
+pub enum HitsInfo {
+    #[serde(rename_all = "camelCase")]
+    Pagination {
+        hits_per_page: usize,
+        page: usize,
+        total_pages: usize,
+        total_hits: usize,
+    },
+    #[serde(rename_all = "camelCase")]
+    OffsetLimit {
+        limit: usize,
+        offset: usize,
+        estimated_total_hits: usize,
+    },
+}
+
 impl Index {
     pub fn perform_search(&self, query: SearchQuery) -> Result<SearchResult> {
         let before_search = Instant::now();
@@ -97,8 +119,30 @@ impl Index {
 
         // Make sure that a user can't get more documents than the hard limit,
         // we align that on the offset too.
-        let offset = min(query.offset.unwrap_or(0), max_total_hits);
-        let limit = min(query.limit, max_total_hits.saturating_sub(offset));
+        let is_finite_pagination = query.offset.is_none() && query.limit.is_none();
+
+        search.exhaustive_number_hits(is_finite_pagination);
+
+        let (offset, limit) = if is_finite_pagination {
+            match query.page.checked_sub(1) {
+                Some(page) => {
+                    let offset = min(query.hits_per_page * page, max_total_hits);
+                    let limit = min(query.hits_per_page, max_total_hits.saturating_sub(offset));
+
+                    (offset, limit)
+                }
+                // page 0 returns 0 hits
+                None => (0, 0),
+            }
+        } else {
+            let offset = min(query.offset.unwrap_or(0), max_total_hits);
+            let limit = min(
+                query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+                max_total_hits.saturating_sub(offset),
+            );
+
+            (offset, limit)
+        };
 
         search.offset(offset);
         search.limit(limit);
@@ -223,7 +267,26 @@ impl Index {
             documents.push(hit);
         }
 
-        let estimated_total_hits = candidates.len();
+        let number_of_hits = min(candidates.len() as usize, max_total_hits);
+        let hits_info = if is_finite_pagination {
+            // If hit_per_page is 0, then pages can't be computed and so we respond 0.
+            let total_pages = (number_of_hits + query.hits_per_page.saturating_sub(1))
+                .checked_div(query.hits_per_page)
+                .unwrap_or(0);
+
+            HitsInfo::Pagination {
+                hits_per_page: query.hits_per_page,
+                page: query.page,
+                total_pages,
+                total_hits: number_of_hits,
+            }
+        } else {
+            HitsInfo::OffsetLimit {
+                limit: query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+                offset,
+                estimated_total_hits: number_of_hits,
+            }
+        };
 
         let facet_distribution = match query.facets {
             Some(ref fields) => {
@@ -246,10 +309,8 @@ impl Index {
 
         let result = SearchResult {
             hits: documents,
-            estimated_total_hits,
+            hits_info,
             query: query.q.clone().unwrap_or_default(),
-            limit: query.limit,
-            offset: query.offset.unwrap_or_default(),
             processing_time_ms: before_search.elapsed().as_millis(),
             facet_distribution,
         };

meilisearch-lib/src/index_controller/mod.rs

@@ -659,7 +659,7 @@ mod test {
     use nelson::Mocker;
 
     use crate::index::error::Result as IndexResult;
-    use crate::index::Index;
+    use crate::index::{HitsInfo, Index};
     use crate::index::{
         DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
     };
@@ -692,7 +692,9 @@ mod test {
         let query = SearchQuery {
             q: Some(String::from("hello world")),
             offset: Some(10),
-            limit: 0,
+            limit: Some(0),
+            page: 1,
+            hits_per_page: 10,
             attributes_to_retrieve: Some(vec!["string".to_owned()].into_iter().collect()),
             attributes_to_crop: None,
             crop_length: 18,
@@ -708,10 +710,12 @@ mod test {
 
         let result = SearchResult {
             hits: vec![],
-            estimated_total_hits: 29,
             query: "hello world".to_string(),
-            limit: 24,
-            offset: 0,
+            hits_info: HitsInfo::OffsetLimit {
+                limit: 24,
+                offset: 0,
+                estimated_total_hits: 29,
+            },
             processing_time_ms: 50,
             facet_distribution: None,
         };