diff --git a/index-scheduler/src/index_mapper.rs b/index-scheduler/src/index_mapper.rs index 13ffb0dc3..7dcdd68c9 100644 --- a/index-scheduler/src/index_mapper.rs +++ b/index-scheduler/src/index_mapper.rs @@ -35,7 +35,10 @@ pub struct IndexMapper { /// Path to the folder where the LMDB environments of each index are. base_path: PathBuf, - index_size: usize, + /// The map size an index is opened with on the first time. + index_base_map_size: usize, + /// The quantity by which the map size of an index is incremented upon reopening, in bytes. + index_growth_amount: usize, pub indexer_config: Arc, } @@ -171,14 +174,17 @@ impl IndexMapper { pub fn new( env: &Env, base_path: PathBuf, - index_size: usize, + index_base_map_size: usize, + index_growth_amount: usize, + index_count: usize, indexer_config: IndexerConfig, ) -> Result { Ok(Self { - index_map: Arc::new(RwLock::new(IndexMap::new(20))), + index_map: Arc::new(RwLock::new(IndexMap::new(index_count))), index_mapping: env.create_database(Some(INDEX_MAPPING))?, base_path, - index_size, + index_base_map_size, + index_growth_amount, indexer_config: Arc::new(indexer_config), }) } @@ -221,7 +227,8 @@ impl IndexMapper { let index_path = self.base_path.join(uuid.to_string()); fs::create_dir_all(&index_path)?; - let index = self.create_or_open_index(&index_path, date, self.index_size)?; + let index = + self.create_or_open_index(&index_path, date, self.index_base_map_size)?; wtxn.commit()?; // Error if the UUIDv4 somehow already exists in the map, since it should be fresh. @@ -324,7 +331,7 @@ impl IndexMapper { let resize_succeeded = (move || { let current_size = index.map_size()?; - let new_size = current_size * 2; + let new_size = current_size + self.index_growth_amount; let closing_event = index.prepare_for_closing(); log::debug!("Waiting for index {name} to close"); @@ -400,8 +407,11 @@ impl IndexMapper { None => { let index_path = self.base_path.join(uuid.to_string()); - let index = - self.create_or_open_index(&index_path, None, self.index_size)?; + let index = self.create_or_open_index( + &index_path, + None, + self.index_base_map_size, + )?; match index_map.insert(&uuid, index.clone()) { InsertionOutcome::InsertedNew => break (index, None), InsertionOutcome::Evicted(evicted_uuid, evicted_index) => { diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index aed3ae879..ea82fe6cd 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -231,8 +231,12 @@ pub struct IndexSchedulerOptions { pub dumps_path: PathBuf, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, - /// The maximum size, in bytes, of each meilisearch index. - pub index_size: usize, + /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. + pub index_base_map_size: usize, + /// The size, in bytes, by which the map size of an index is increased when it resized due to being full. + pub index_growth_amount: usize, + /// The number of indexes that can be concurrently opened in memory. + pub index_count: usize, /// Configuration used during indexing for each meilisearch index. pub indexer_config: IndexerConfig, /// Set to `true` iff the index scheduler is allowed to automatically @@ -384,7 +388,9 @@ impl IndexScheduler { index_mapper: IndexMapper::new( &env, options.indexes_path, - options.index_size, + options.index_base_map_size, + options.index_growth_amount, + options.index_count, options.indexer_config, )?, env, @@ -1164,7 +1170,9 @@ mod tests { snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. - index_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. + index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. + index_growth_amount: 1000 * 1000, // 1 MB + index_count: 5, indexer_config: IndexerConfig::default(), autobatching_enabled, }; diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index f17f93e74..3515e88a4 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -205,9 +205,11 @@ fn open_or_create_database_unchecked( snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), task_db_size: opt.max_task_db_size.get_bytes() as usize, - index_size: opt.max_index_size.get_bytes() as usize, + index_base_map_size: opt.max_index_size.get_bytes() as usize, indexer_config: (&opt.indexer_options).try_into()?, autobatching_enabled: true, + index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize, + index_count: 20, })?) };