diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index 04020a74f..4635c572f 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -205,6 +205,8 @@ struct Infos { experimental_no_snapshot_compaction: bool, experimental_no_edition_2024_for_dumps: bool, experimental_no_edition_2024_for_settings: bool, + experimental_no_edition_2024_for_prefix_post_processing: bool, + experimental_no_edition_2024_for_facet_post_processing: bool, experimental_vector_store_setting: bool, gpu_enabled: bool, db_path: bool, @@ -296,6 +298,8 @@ impl Infos { skip_index_budget: _, experimental_no_edition_2024_for_settings, experimental_no_edition_2024_for_dumps, + experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing, } = indexer_options; let RuntimeTogglableFeatures { @@ -365,6 +369,8 @@ impl Infos { ssl_resumption, ssl_tickets, experimental_no_edition_2024_for_settings, + experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing, } } } diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index 803fecf01..17522e493 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -55,6 +55,10 @@ const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LO const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER"; const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS: &str = "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS"; +const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_FACET_POST_PROCESSING: &str = + "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_FACET_POST_PROCESSING"; +const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_PREFIX_POST_PROCESSING: &str = + "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_PREFIX_POST_PROCESSING"; const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS"; const MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE: &str = "MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE"; const MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER: &str = "MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER"; @@ -772,6 +776,22 @@ pub struct IndexerOpts { #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS)] #[serde(default)] pub experimental_no_edition_2024_for_dumps: bool, + + /// Experimental no edition 2024 to compute prefixes. For more information, + /// see: + /// + /// Enables the experimental no edition 2024 to compute prefixes. + #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_PREFIX_POST_PROCESSING)] + #[serde(default)] + pub experimental_no_edition_2024_for_prefix_post_processing: bool, + + /// Experimental no edition 2024 to compute facets. For more information, + /// see: + /// + /// Enables the experimental no edition 2024 to compute facets. + #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_FACET_POST_PROCESSING)] + #[serde(default)] + pub experimental_no_edition_2024_for_facet_post_processing: bool, } impl IndexerOpts { @@ -783,6 +803,8 @@ impl IndexerOpts { skip_index_budget: _, experimental_no_edition_2024_for_settings, experimental_no_edition_2024_for_dumps, + experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing, } = self; if let Some(max_indexing_memory) = max_indexing_memory.0 { export_to_env_if_not_present( @@ -808,6 +830,18 @@ impl IndexerOpts { experimental_no_edition_2024_for_dumps.to_string(), ); } + if experimental_no_edition_2024_for_prefix_post_processing { + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_PREFIX_POST_PROCESSING, + experimental_no_edition_2024_for_prefix_post_processing.to_string(), + ); + } + if experimental_no_edition_2024_for_facet_post_processing { + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_FACET_POST_PROCESSING, + experimental_no_edition_2024_for_facet_post_processing.to_string(), + ); + } } } @@ -833,6 +867,10 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { chunk_compression_level: Default::default(), documents_chunk_size: Default::default(), max_nb_chunks: Default::default(), + experimental_no_edition_2024_for_prefix_post_processing: other + .experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing: other + .experimental_no_edition_2024_for_facet_post_processing, }) } } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 291356bf8..b87dbe0ad 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -490,6 +490,8 @@ pub fn default_settings(dir: impl AsRef) -> Opt { max_indexing_threads: MaxThreads::from_str("2").unwrap(), experimental_no_edition_2024_for_settings: false, experimental_no_edition_2024_for_dumps: false, + experimental_no_edition_2024_for_prefix_post_processing: false, + experimental_no_edition_2024_for_facet_post_processing: false, }, experimental_enable_metrics: false, ..Parser::parse_from(None as Option<&str>) diff --git a/crates/milli/src/update/index_documents/helpers/grenad_helpers.rs b/crates/milli/src/update/index_documents/helpers/grenad_helpers.rs index 62dc40edc..7b53ee2b5 100644 --- a/crates/milli/src/update/index_documents/helpers/grenad_helpers.rs +++ b/crates/milli/src/update/index_documents/helpers/grenad_helpers.rs @@ -101,6 +101,8 @@ pub struct GrenadParameters { pub chunk_compression_level: Option, pub max_memory: Option, pub max_nb_chunks: Option, + pub experimental_no_edition_2024_for_prefix_post_processing: bool, + pub experimental_no_edition_2024_for_facet_post_processing: bool, } impl Default for GrenadParameters { @@ -110,6 +112,8 @@ impl Default for GrenadParameters { chunk_compression_level: None, max_memory: None, max_nb_chunks: None, + experimental_no_edition_2024_for_prefix_post_processing: false, + experimental_no_edition_2024_for_facet_post_processing: false, } } } diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 857afdee5..97f1afb8e 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -254,6 +254,12 @@ where chunk_compression_level: self.indexer_config.chunk_compression_level, max_memory: self.indexer_config.max_memory, max_nb_chunks: self.indexer_config.max_nb_chunks, // default value, may be chosen. + experimental_no_edition_2024_for_prefix_post_processing: self + .indexer_config + .experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing: self + .indexer_config + .experimental_no_edition_2024_for_facet_post_processing, }; let documents_chunk_size = match self.indexer_config.documents_chunk_size { Some(chunk_size) => chunk_size, diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index 53373eaec..ef7f2fedc 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -983,6 +983,12 @@ impl<'a, 'i> Transform<'a, 'i> { chunk_compression_level: self.indexer_settings.chunk_compression_level, max_memory: self.indexer_settings.max_memory, max_nb_chunks: self.indexer_settings.max_nb_chunks, // default value, may be chosen. + experimental_no_edition_2024_for_prefix_post_processing: self + .indexer_settings + .experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing: self + .indexer_settings + .experimental_no_edition_2024_for_facet_post_processing, }; // Once we have written all the documents, we merge everything into a Reader. diff --git a/crates/milli/src/update/indexer_config.rs b/crates/milli/src/update/indexer_config.rs index 39a013d13..65e32483d 100644 --- a/crates/milli/src/update/indexer_config.rs +++ b/crates/milli/src/update/indexer_config.rs @@ -18,6 +18,8 @@ pub struct IndexerConfig { pub skip_index_budget: bool, pub experimental_no_edition_2024_for_settings: bool, pub experimental_no_edition_2024_for_dumps: bool, + pub experimental_no_edition_2024_for_prefix_post_processing: bool, + pub experimental_no_edition_2024_for_facet_post_processing: bool, } impl IndexerConfig { @@ -27,6 +29,10 @@ impl IndexerConfig { chunk_compression_level: self.chunk_compression_level, max_memory: self.max_memory, max_nb_chunks: self.max_nb_chunks, + experimental_no_edition_2024_for_prefix_post_processing: self + .experimental_no_edition_2024_for_prefix_post_processing, + experimental_no_edition_2024_for_facet_post_processing: self + .experimental_no_edition_2024_for_facet_post_processing, } } } @@ -68,6 +74,8 @@ impl Default for IndexerConfig { skip_index_budget: false, experimental_no_edition_2024_for_settings: false, experimental_no_edition_2024_for_dumps: false, + experimental_no_edition_2024_for_prefix_post_processing: false, + experimental_no_edition_2024_for_facet_post_processing: false, } } } diff --git a/crates/milli/src/update/new/indexer/post_processing/mod.rs b/crates/milli/src/update/new/indexer/post_processing/mod.rs index 71a02c207..762d213a8 100644 --- a/crates/milli/src/update/new/indexer/post_processing/mod.rs +++ b/crates/milli/src/update/new/indexer/post_processing/mod.rs @@ -42,6 +42,7 @@ where wtxn, facet_field_ids_delta, &mut global_fields_ids_map, + indexing_context.grenad_parameters, indexing_context.progress, )?; compute_facet_search_database(index, wtxn, global_fields_ids_map, indexing_context.progress)?; @@ -219,6 +220,7 @@ fn compute_facet_level_database( wtxn: &mut RwTxn, mut facet_field_ids_delta: FacetFieldIdsDelta, global_fields_ids_map: &mut GlobalFieldsIdsMap, + grenad_parameters: &GrenadParameters, progress: &Progress, ) -> Result<()> { let rtxn = index.read_txn()?; @@ -242,8 +244,14 @@ fn compute_facet_level_database( match delta { FacetFieldIdDelta::Bulk => { progress.update_progress(PostProcessingFacets::StringsBulk); - tracing::debug!(%fid, "bulk string facet processing in parallel"); - generate_facet_levels(index, wtxn, fid, FacetType::String)? + if grenad_parameters.experimental_no_edition_2024_for_facet_post_processing { + tracing::debug!(%fid, "bulk string facet processing"); + FacetsUpdateBulk::new_not_updating_level_0(index, vec![fid], FacetType::String) + .execute(wtxn)? + } else { + tracing::debug!(%fid, "bulk string facet processing in parallel"); + generate_facet_levels(index, wtxn, fid, FacetType::String)? + } } FacetFieldIdDelta::Incremental(delta_data) => { progress.update_progress(PostProcessingFacets::StringsIncremental); diff --git a/crates/milli/src/update/new/words_prefix_docids.rs b/crates/milli/src/update/new/words_prefix_docids.rs index 9518dab8e..ee4340674 100644 --- a/crates/milli/src/update/new/words_prefix_docids.rs +++ b/crates/milli/src/update/new/words_prefix_docids.rs @@ -157,7 +157,8 @@ struct WordPrefixIntegerDocids<'i> { database: Database, prefix_database: Database, max_memory_by_thread: Option, - read_uncommitted_in_parallel: bool, + /// Do not use an experimental LMDB feature to read uncommitted data in parallel. + no_experimental_post_processing: bool, } impl<'i> WordPrefixIntegerDocids<'i> { @@ -172,15 +173,11 @@ impl<'i> WordPrefixIntegerDocids<'i> { database, prefix_database, max_memory_by_thread: grenad_parameters.max_memory_by_thread(), - read_uncommitted_in_parallel: false, + no_experimental_post_processing: grenad_parameters + .experimental_no_edition_2024_for_prefix_post_processing, } } - /// Use an experimental LMDB feature to read uncommitted data in parallel. - fn read_uncommitted_in_parallel(&mut self, value: bool) { - self.read_uncommitted_in_parallel = value; - } - fn execute( self, wtxn: &mut heed::RwTxn, @@ -188,10 +185,10 @@ impl<'i> WordPrefixIntegerDocids<'i> { prefix_to_delete: &BTreeSet, ) -> Result<()> { delete_prefixes(wtxn, &self.prefix_database, prefix_to_delete)?; - if self.read_uncommitted_in_parallel { - self.recompute_modified_prefixes_no_frozen(wtxn, prefix_to_compute) - } else { + if self.no_experimental_post_processing { self.recompute_modified_prefixes(wtxn, prefix_to_compute) + } else { + self.recompute_modified_prefixes_no_frozen(wtxn, prefix_to_compute) } } @@ -510,14 +507,13 @@ pub fn compute_word_prefix_fid_docids( prefix_to_delete: &BTreeSet, grenad_parameters: &GrenadParameters, ) -> Result<()> { - let mut builder = WordPrefixIntegerDocids::new( + WordPrefixIntegerDocids::new( index, index.word_fid_docids.remap_key_type(), index.word_prefix_fid_docids.remap_key_type(), grenad_parameters, - ); - builder.read_uncommitted_in_parallel(true); - builder.execute(wtxn, prefix_to_compute, prefix_to_delete) + ) + .execute(wtxn, prefix_to_compute, prefix_to_delete) } #[tracing::instrument(level = "trace", skip_all, target = "indexing::prefix")] @@ -528,12 +524,11 @@ pub fn compute_word_prefix_position_docids( prefix_to_delete: &BTreeSet, grenad_parameters: &GrenadParameters, ) -> Result<()> { - let mut builder = WordPrefixIntegerDocids::new( + WordPrefixIntegerDocids::new( index, index.word_position_docids.remap_key_type(), index.word_prefix_position_docids.remap_key_type(), grenad_parameters, - ); - builder.read_uncommitted_in_parallel(true); - builder.execute(wtxn, prefix_to_compute, prefix_to_delete) + ) + .execute(wtxn, prefix_to_compute, prefix_to_delete) }