snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-27 08:41:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Take the BBQueue capacity into account in the max memory

Browse Source
This commit is contained in:
Clément Renault
2024-11-28 15:43:14 +01:00
parent b57dd5c58e
commit 3c7ac093d3
2 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
crates/milli/src/update/new/channel.rs
View File

@ -27,8 +27,9 @@ use crate::{CboRoaringBitmapCodec, DocumentId, Index};
/// Creates a tuple of senders/receiver to be used by
/// the extractors and the writer loop.
///
/// The `bbqueue_capacity` represent the number of bytes allocated
/// to each BBQueue buffer and is not the sum of all of them.
/// The `total_bbbuffer_capacity` represent the number of bytes
/// allocated to all BBQueue buffer. It will be split by the
/// number of thread.
///
/// The `channel_capacity` parameter defines the number of
/// too-large-to-fit-in-BBQueue entries that can be sent through
@ -46,10 +47,12 @@ use crate::{CboRoaringBitmapCodec, DocumentId, Index};
/// to the number of available threads in the rayon threadpool.
pub fn extractor_writer_bbqueue(
bbbuffers: &mut Vec<BBBuffer>,
bbbuffer_capacity: usize,
total_bbbuffer_capacity: usize,
channel_capacity: usize,
) -> (ExtractorBbqueueSender, WriterBbqueueReceiver) {
bbbuffers.resize_with(rayon::current_num_threads(), || BBBuffer::new(bbbuffer_capacity));
let current_num_threads = rayon::current_num_threads();
let bbbuffer_capacity = total_bbbuffer_capacity.checked_div(current_num_threads).unwrap();
bbbuffers.resize_with(current_num_threads, || BBBuffer::new(bbbuffer_capacity));
let capacity = bbbuffers.first().unwrap().capacity();
// Read the field description to understand this