Fix tests

Cargo.lock

@@ -644,9 +644,9 @@ dependencies = [
 
 [[package]]
 name = "charabia"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a26a3df4d9c9231eb1e757fe6b1c66c471e0c2cd5410265e7c3109a726663c4"
+checksum = "2ed19edcd98f5bf6572f48d6f5982d595cb8718e47c6f0066d942b280575ff02"
 dependencies = [
  "character_converter",
  "cow-utils",
@@ -1124,7 +1124,7 @@ dependencies = [
 [[package]]
 name = "filter-parser"
 version = "0.31.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#ff97a6017c9e409b46df68930b0d2774159b29ad"
 dependencies = [
  "nom",
  "nom_locate",
@@ -1149,7 +1149,7 @@ dependencies = [
 [[package]]
 name = "flatten-serde-json"
 version = "0.31.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#ff97a6017c9e409b46df68930b0d2774159b29ad"
 dependencies = [
  "serde_json",
 ]
@@ -1662,7 +1662,7 @@ dependencies = [
 [[package]]
 name = "json-depth-checker"
 version = "0.31.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#ff97a6017c9e409b46df68930b0d2774159b29ad"
 dependencies = [
  "serde_json",
 ]
@@ -2013,7 +2013,7 @@ dependencies = [
  "sha2",
  "thiserror",
  "time 0.3.9",
- "uuid 1.1.2",
+ "uuid",
 ]
 
 [[package]]
@@ -2082,7 +2082,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "urlencoding",
- "uuid 1.1.2",
+ "uuid",
  "vergen",
  "walkdir",
  "yaup",
@@ -2146,7 +2146,7 @@ dependencies = [
  "thiserror",
  "time 0.3.9",
  "tokio",
- "uuid 1.1.2",
+ "uuid",
  "walkdir",
  "whoami",
 ]
@@ -2189,7 +2189,7 @@ dependencies = [
 [[package]]
 name = "milli"
 version = "0.31.1"
-source = "git+https://github.com/meilisearch/milli.git?tag=v0.31.1#83ad1aaf0552db9f63fc21ae9fe3976e61577dc8"
+source = "git+https://github.com/meilisearch/milli.git?branch=ease-search-results-pagination#ff97a6017c9e409b46df68930b0d2774159b29ad"
 dependencies = [
  "bimap",
  "bincode",
@@ -2228,7 +2228,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "time 0.3.9",
- "uuid 0.8.2",
+ "uuid",
 ]
 
 [[package]]
@@ -3670,15 +3670,6 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1"
 
-[[package]]
-name = "uuid"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
-dependencies = [
- "getrandom",
-]
-
 [[package]]
 name = "uuid"
 version = "1.1.2"

meilisearch-auth/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 enum-iterator = "0.7.0"
 hmac = "0.12.1"
 meilisearch-types = { path = "../meilisearch-types" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" }
+milli = { git = "https://github.com/meilisearch/milli.git", branch = "ease-search-results-pagination" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { version = "1.0.79", features = ["preserve_order"] }

meilisearch-http/src/analytics/segment_analytics.rs

@@ -10,7 +10,7 @@ use http::header::CONTENT_TYPE;
 use meilisearch_auth::SearchRules;
 use meilisearch_lib::index::{
     SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
 };
 use meilisearch_lib::index_controller::Stats;
 use meilisearch_lib::MeiliSearch;
@@ -369,6 +369,7 @@ pub struct SearchAggregator {
     // pagination
     max_limit: usize,
     max_offset: usize,
+    finite_pagination: bool,
 
     // formatting
     highlight_pre_tag: bool,
@@ -423,8 +424,15 @@ impl SearchAggregator {
             ret.max_terms_number = q.split_whitespace().count();
         }
 
-        ret.max_limit = query.limit;
-        ret.max_offset = query.offset.unwrap_or_default();
+        if query.limit.is_none() && query.offset.is_none() {
+            ret.max_limit = query.hits_per_page;
+            ret.max_offset = query.page.saturating_sub(1) * query.hits_per_page;
+            ret.finite_pagination = true;
+        } else {
+            ret.max_limit = query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
+            ret.max_offset = query.offset.unwrap_or_default();
+            ret.finite_pagination = false;
+        }
 
         ret.highlight_pre_tag = query.highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG();
         ret.highlight_post_tag = query.highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG();

meilisearch-http/src/routes/indexes/search.rs

@@ -3,7 +3,7 @@ use log::debug;
 use meilisearch_auth::IndexSearchRules;
 use meilisearch_lib::index::{
     SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_HIT_PER_PAGE, DEFAULT_PAGE,
 };
 use meilisearch_lib::MeiliSearch;
 use meilisearch_types::error::ResponseError;
@@ -29,6 +29,10 @@ pub struct SearchQueryGet {
     q: Option<String>,
     offset: Option<usize>,
     limit: Option<usize>,
+    #[serde(default = "DEFAULT_PAGE")]
+    page: usize,
+    #[serde(default = "DEFAULT_HIT_PER_PAGE")]
+    hits_per_page: usize,
     attributes_to_retrieve: Option<CS<String>>,
     attributes_to_crop: Option<CS<String>>,
     #[serde(default = "DEFAULT_CROP_LENGTH")]
@@ -60,7 +64,9 @@ impl From<SearchQueryGet> for SearchQuery {
         Self {
             q: other.q,
             offset: other.offset,
-            limit: other.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+            limit: other.limit,
+            page: other.page,
+            hits_per_page: other.hits_per_page,
             attributes_to_retrieve: other
                 .attributes_to_retrieve
                 .map(|o| o.into_iter().collect()),

meilisearch-lib/Cargo.toml

@@ -28,7 +28,7 @@ lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.1" }
+milli = { git = "https://github.com/meilisearch/milli.git", branch = "ease-search-results-pagination" }
 mime = "0.3.16"
 num_cpus = "1.13.1"
 obkv = "0.2.0"

meilisearch-lib/src/error.rs

@@ -25,6 +25,8 @@ impl ErrorCode for MilliError<'_> {
                     // TODO: wait for spec for new error codes.
                     UserError::SerdeJson(_)
                     | UserError::DocumentLimitReached
+                    // TODO: REMOVE ME
+                    | UserError::AccessingSoftDeletedDocument { .. }
                     | UserError::UnknownInternalDocumentId { .. } => Code::Internal,
                     UserError::InvalidStoreFile => Code::InvalidStore,
                     UserError::NoSpaceLeftOnDevice => Code::NoSpaceLeftOnDevice,

meilisearch-lib/src/index/mod.rs

@@ -1,6 +1,7 @@
 pub use search::{
-    SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    HitsInfo, SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_HIT_PER_PAGE, DEFAULT_PAGE,
+    DEFAULT_SEARCH_LIMIT,
 };
 pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
 

meilisearch-lib/src/index/search.rs

@@ -1,4 +1,4 @@
-use std::cmp::min;
+use std::cmp::{max, min};
 use std::collections::{BTreeMap, BTreeSet, HashSet};
 use std::str::FromStr;
 use std::time::Instant;
@@ -26,6 +26,8 @@ pub const DEFAULT_CROP_LENGTH: fn() -> usize = || 10;
 pub const DEFAULT_CROP_MARKER: fn() -> String = || "…".to_string();
 pub const DEFAULT_HIGHLIGHT_PRE_TAG: fn() -> String = || "<em>".to_string();
 pub const DEFAULT_HIGHLIGHT_POST_TAG: fn() -> String = || "</em>".to_string();
+pub const DEFAULT_PAGE: fn() -> usize = || 1;
+pub const DEFAULT_HIT_PER_PAGE: fn() -> usize = || 20;
 
 /// The maximimum number of results that the engine
 /// will be able to return in one search call.
@@ -36,8 +38,11 @@ pub const DEFAULT_PAGINATION_MAX_TOTAL_HITS: usize = 1000;
 pub struct SearchQuery {
     pub q: Option<String>,
     pub offset: Option<usize>,
-    #[serde(default = "DEFAULT_SEARCH_LIMIT")]
-    pub limit: usize,
+    pub limit: Option<usize>,
+    #[serde(default = "DEFAULT_PAGE")]
+    pub page: usize,
+    #[serde(default = "DEFAULT_HIT_PER_PAGE")]
+    pub hits_per_page: usize,
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
     pub attributes_to_crop: Option<Vec<String>>,
     #[serde(default = "DEFAULT_CROP_LENGTH")]
@@ -71,15 +76,31 @@ pub struct SearchHit {
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
-    pub estimated_total_hits: u64,
     pub query: String,
-    pub limit: usize,
-    pub offset: usize,
     pub processing_time_ms: u128,
+    #[serde(flatten)]
+    pub hits_info: HitsInfo,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub facet_distribution: Option<BTreeMap<String, BTreeMap<String, u64>>>,
 }
 
+#[derive(Serialize, Debug, Clone, PartialEq)]
+#[serde(untagged)]
+pub enum HitsInfo {
+    #[serde(rename_all = "camelCase")]
+    Pagination {
+        hits_per_page: usize,
+        page: usize,
+        total_pages: usize,
+    },
+    #[serde(rename_all = "camelCase")]
+    OffsetLimit {
+        limit: usize,
+        offset: usize,
+        estimated_total_hits: usize,
+    },
+}
+
 impl Index {
     pub fn perform_search(&self, query: SearchQuery) -> Result<SearchResult> {
         let before_search = Instant::now();
@@ -97,8 +118,28 @@ impl Index {
 
         // Make sure that a user can't get more documents than the hard limit,
         // we align that on the offset too.
-        let offset = min(query.offset.unwrap_or(0), max_total_hits);
-        let limit = min(query.limit, max_total_hits.saturating_sub(offset));
+        let is_finite_pagination = query.offset.is_none() && query.limit.is_none();
+
+        search.exhaustive_number_hits(is_finite_pagination);
+
+        let (offset, limit) = if is_finite_pagination {
+            // we start at least at page 1.
+            let page = max(query.page, 1);
+            // return at least 1 document.
+            let hits_per_page = max(query.hits_per_page, 1);
+            let offset = min(hits_per_page * (page - 1), max_total_hits);
+            let limit = min(hits_per_page, max_total_hits.saturating_sub(offset));
+
+            (offset, limit)
+        } else {
+            let offset = min(query.offset.unwrap_or(0), max_total_hits);
+            let limit = min(
+                query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+                max_total_hits.saturating_sub(offset),
+            );
+
+            (offset, limit)
+        };
 
         search.offset(offset);
         search.limit(limit);
@@ -223,7 +264,23 @@ impl Index {
             documents.push(hit);
         }
 
-        let estimated_total_hits = candidates.len();
+        let number_of_hits = min(candidates.len() as usize, max_total_hits);
+        let hits_info = if is_finite_pagination {
+            // return at least 1 document.
+            let hits_per_page = max(query.hits_per_page, 1);
+            HitsInfo::Pagination {
+                hits_per_page,
+                page: offset / hits_per_page + 1,
+                // TODO @many: estimation for now but we should ask milli to return an exact value
+                total_pages: (number_of_hits + hits_per_page - 1) / query.hits_per_page,
+            }
+        } else {
+            HitsInfo::OffsetLimit {
+                limit: query.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+                offset,
+                estimated_total_hits: number_of_hits,
+            }
+        };
 
         let facet_distribution = match query.facets {
             Some(ref fields) => {
@@ -246,10 +303,8 @@ impl Index {
 
         let result = SearchResult {
             hits: documents,
-            estimated_total_hits,
+            hits_info,
             query: query.q.clone().unwrap_or_default(),
-            limit: query.limit,
-            offset: query.offset.unwrap_or_default(),
             processing_time_ms: before_search.elapsed().as_millis(),
             facet_distribution,
         };

meilisearch-lib/src/index_controller/mod.rs

@@ -652,7 +652,7 @@ mod test {
     use nelson::Mocker;
 
     use crate::index::error::Result as IndexResult;
-    use crate::index::Index;
+    use crate::index::{HitsInfo, Index};
     use crate::index::{
         DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
     };
@@ -685,7 +685,9 @@ mod test {
         let query = SearchQuery {
             q: Some(String::from("hello world")),
             offset: Some(10),
-            limit: 0,
+            limit: Some(0),
+            page: 1,
+            hits_per_page: 10,
             attributes_to_retrieve: Some(vec!["string".to_owned()].into_iter().collect()),
             attributes_to_crop: None,
             crop_length: 18,
@@ -701,10 +703,12 @@ mod test {
 
         let result = SearchResult {
             hits: vec![],
-            estimated_total_hits: 29,
             query: "hello world".to_string(),
-            limit: 24,
-            offset: 0,
+            hits_info: HitsInfo::OffsetLimit {
+                limit: 24,
+                offset: 0,
+                estimated_total_hits: 29,
+            },
             processing_time_ms: 50,
             facet_distribution: None,
         };