Merge #5141

5141: Use the right amount of max memory and not impact the settings r=curquiza a=Kerollmops

Fixes #5132. Related to #5125.

Co-authored-by: Kerollmops <clement@meilisearch.com>

crates/meilisearch/src/option.rs

@@ -654,9 +654,8 @@ impl Opt {
 
 #[derive(Debug, Default, Clone, Parser, Deserialize)]
 pub struct IndexerOpts {
-    /// Specifies the maximum resident memory that Meilisearch can use for indexing.
+    /// Sets the maximum amount of RAM Meilisearch can use when indexing. By default, Meilisearch
-    /// By default, Meilisearch limits the RAM usage to 5% of the total available memory.
+    /// uses no more than two thirds of available memory.
-    /// Note that the underlying store utilizes memory-mapping and makes use of the rest.
     #[clap(long, env = MEILI_MAX_INDEXING_MEMORY, default_value_t)]
     #[serde(default)]
     pub max_indexing_memory: MaxMemory,
@@ -715,7 +714,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
     }
 }
 
-/// A type used to detect the max resident memory available and use 5% of it.
+/// A type used to detect the max memory available and use 2/3 of it.
 #[derive(Debug, Clone, Copy, Deserialize, Serialize)]
 pub struct MaxMemory(Option<Byte>);
 
@@ -729,7 +728,7 @@ impl FromStr for MaxMemory {
 
 impl Default for MaxMemory {
     fn default() -> MaxMemory {
-        MaxMemory(total_memory_bytes().map(|bytes| bytes * 5 / 100).map(Byte::from_u64))
+        MaxMemory(total_memory_bytes().map(|bytes| bytes * 2 / 3).map(Byte::from_u64))
     }
 }
 

crates/milli/src/update/new/indexer/mod.rs

@@ -80,6 +80,15 @@ where
     let mut bbbuffers = Vec::new();
     let finished_extraction = AtomicBool::new(false);
 
+    // We reduce the actual memory used to 5%. The reason we do this here and not in Meilisearch
+    // is because we still use the old indexer for the settings and it is highly impacted by the
+    // max memory. So we keep the changes here and will remove these changes once we use the new
+    // indexer to also index settings. Related to #5125 and #5141.
+    let grenad_parameters = GrenadParameters {
+        max_memory: grenad_parameters.max_memory.map(|mm| mm * 5 / 100),
+        ..grenad_parameters
+    };
+
     // We compute and remove the allocated BBQueues buffers capacity from the indexing memory.
     let minimum_capacity = 50 * 1024 * 1024 * pool.current_num_threads(); // 50 MiB
     let (grenad_parameters, total_bbbuffer_capacity) = grenad_parameters.max_memory.map_or(