mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-06-10 14:07:58 +00:00
refactor: remove runtime cfg!(test) check
Won't work in integration tests and consequently all threads would be used. To remedy this we make explicit `max_threads=Some(1)` in the IndexerConfig::default
This commit is contained in:
parent
75a7e40a27
commit
15cdc6924b
@ -501,12 +501,11 @@ fn import_dump(
|
|||||||
let network = dump_reader.network()?.cloned().unwrap_or_default();
|
let network = dump_reader.network()?.cloned().unwrap_or_default();
|
||||||
index_scheduler.put_network(network)?;
|
index_scheduler.put_network(network)?;
|
||||||
|
|
||||||
// 3.1 Use all cpus to process dump if a) `max_indexing_threads` not configured and
|
// 3.1 Use all cpus to process dump if `max_indexing_threads` not configured
|
||||||
// b) we're not executing from within a test
|
|
||||||
let backup_config;
|
let backup_config;
|
||||||
let base_config = index_scheduler.indexer_config();
|
let base_config = index_scheduler.indexer_config();
|
||||||
|
|
||||||
let indexer_config = if base_config.max_threads.is_none() && !cfg!(test) {
|
let indexer_config = if base_config.max_threads.is_none() {
|
||||||
let thread_pool = ThreadPoolNoAbortBuilder::new()
|
let thread_pool = ThreadPoolNoAbortBuilder::new()
|
||||||
.thread_name(|index| format!("indexing-thread:{index}"))
|
.thread_name(|index| format!("indexing-thread:{index}"))
|
||||||
.num_threads(num_cpus::get())
|
.num_threads(num_cpus::get())
|
||||||
|
@ -19,10 +19,7 @@ use crate::update::{
|
|||||||
};
|
};
|
||||||
use crate::vector::settings::{EmbedderSource, EmbeddingSettings};
|
use crate::vector::settings::{EmbedderSource, EmbeddingSettings};
|
||||||
use crate::vector::EmbeddingConfigs;
|
use crate::vector::EmbeddingConfigs;
|
||||||
use crate::{
|
use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult};
|
||||||
db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult,
|
|
||||||
ThreadPoolNoAbortBuilder,
|
|
||||||
};
|
|
||||||
|
|
||||||
pub(crate) struct TempIndex {
|
pub(crate) struct TempIndex {
|
||||||
pub inner: Index,
|
pub inner: Index,
|
||||||
@ -62,15 +59,8 @@ impl TempIndex {
|
|||||||
wtxn: &mut RwTxn<'t>,
|
wtxn: &mut RwTxn<'t>,
|
||||||
documents: Mmap,
|
documents: Mmap,
|
||||||
) -> Result<(), crate::error::Error> {
|
) -> Result<(), crate::error::Error> {
|
||||||
let local_pool;
|
|
||||||
let indexer_config = &self.indexer_config;
|
let indexer_config = &self.indexer_config;
|
||||||
let pool = match &indexer_config.thread_pool {
|
let pool = &indexer_config.thread_pool;
|
||||||
Some(pool) => pool,
|
|
||||||
None => {
|
|
||||||
local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
|
|
||||||
&local_pool
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let rtxn = self.inner.read_txn()?;
|
let rtxn = self.inner.read_txn()?;
|
||||||
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
|
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
|
||||||
@ -153,15 +143,8 @@ impl TempIndex {
|
|||||||
wtxn: &mut RwTxn<'t>,
|
wtxn: &mut RwTxn<'t>,
|
||||||
external_document_ids: Vec<String>,
|
external_document_ids: Vec<String>,
|
||||||
) -> Result<(), crate::error::Error> {
|
) -> Result<(), crate::error::Error> {
|
||||||
let local_pool;
|
|
||||||
let indexer_config = &self.indexer_config;
|
let indexer_config = &self.indexer_config;
|
||||||
let pool = match &indexer_config.thread_pool {
|
let pool = &indexer_config.thread_pool;
|
||||||
Some(pool) => pool,
|
|
||||||
None => {
|
|
||||||
local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
|
|
||||||
&local_pool
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let rtxn = self.inner.read_txn()?;
|
let rtxn = self.inner.read_txn()?;
|
||||||
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
|
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
|
||||||
@ -231,15 +214,8 @@ fn aborting_indexation() {
|
|||||||
let mut wtxn = index.inner.write_txn().unwrap();
|
let mut wtxn = index.inner.write_txn().unwrap();
|
||||||
let should_abort = AtomicBool::new(false);
|
let should_abort = AtomicBool::new(false);
|
||||||
|
|
||||||
let local_pool;
|
|
||||||
let indexer_config = &index.indexer_config;
|
let indexer_config = &index.indexer_config;
|
||||||
let pool = match &indexer_config.thread_pool {
|
let pool = &indexer_config.thread_pool;
|
||||||
Some(pool) => pool,
|
|
||||||
None => {
|
|
||||||
local_pool = ThreadPoolNoAbortBuilder::new().build().unwrap();
|
|
||||||
&local_pool
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let rtxn = index.inner.read_txn().unwrap();
|
let rtxn = index.inner.read_txn().unwrap();
|
||||||
let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap();
|
let db_fields_ids_map = index.inner.fields_ids_map(&rtxn).unwrap();
|
||||||
|
@ -33,9 +33,13 @@ impl Default for IndexerConfig {
|
|||||||
#[allow(unused_mut)]
|
#[allow(unused_mut)]
|
||||||
let mut pool_builder = ThreadPoolNoAbortBuilder::new();
|
let mut pool_builder = ThreadPoolNoAbortBuilder::new();
|
||||||
|
|
||||||
|
#[allow(unused_mut, unused_assignments)]
|
||||||
|
let mut max_threads = None;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
{
|
{
|
||||||
pool_builder = pool_builder.num_threads(1);
|
pool_builder = pool_builder.num_threads(1);
|
||||||
|
max_threads = Some(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
let thread_pool = pool_builder
|
let thread_pool = pool_builder
|
||||||
@ -44,16 +48,16 @@ impl Default for IndexerConfig {
|
|||||||
.expect("failed to build default rayon thread pool");
|
.expect("failed to build default rayon thread pool");
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
|
max_threads,
|
||||||
|
thread_pool,
|
||||||
log_every_n: None,
|
log_every_n: None,
|
||||||
max_nb_chunks: None,
|
max_nb_chunks: None,
|
||||||
documents_chunk_size: None,
|
documents_chunk_size: None,
|
||||||
max_memory: None,
|
max_memory: None,
|
||||||
max_threads: None,
|
|
||||||
chunk_compression_type: CompressionType::None,
|
chunk_compression_type: CompressionType::None,
|
||||||
chunk_compression_level: None,
|
chunk_compression_level: None,
|
||||||
max_positions_per_attributes: None,
|
max_positions_per_attributes: None,
|
||||||
skip_index_budget: false,
|
skip_index_budget: false,
|
||||||
thread_pool,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user