snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-14 00:36:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Avoid using too much memory when indexing facet-exists-docids

Browse Source
This commit is contained in:
Loïc Lecrenier
2022-07-19 14:42:35 +02:00
parent d0eee5ff7a
commit 1506683705
3 changed files with 40 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs
View File

@ -5,13 +5,15 @@ use std::io;
use std::mem::size_of;
use heed::zerocopy::AsBytes;
use heed::BytesEncode;
use roaring::RoaringBitmap;
use serde_json::Value;
use super::helpers::{create_sorter, keep_first, sorter_into_reader, GrenadParameters};
use crate::error::InternalError;
use crate::facet::value_encoding::f64_into_bytes;
use crate::{DocumentId, FieldId, Result, BEU32};
use crate::update::index_documents::{create_writer, writer_into_reader};
use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32};
/// Extracts the facet values of each faceted field of each document.
///
@ -22,7 +24,7 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
obkv_documents: grenad::Reader<R>,
indexer: GrenadParameters,
faceted_fields: &HashSet<FieldId>,
) -> Result<(grenad::Reader<File>, grenad::Reader<File>, BTreeMap<FieldId, RoaringBitmap>)> {
) -> Result<(grenad::Reader<File>, grenad::Reader<File>, grenad::Reader<File>)> {
let max_memory = indexer.max_memory_by_thread();
let mut fid_docid_facet_numbers_sorter = create_sorter(
@ -91,10 +93,21 @@ pub fn extract_fid_docid_facet_values<R: io::Read + io::Seek>(
}
}
let mut facet_exists_docids_writer = create_writer(
indexer.chunk_compression_type,
indexer.chunk_compression_level,
tempfile::tempfile()?,
);
for (fid, bitmap) in facet_exists_docids.into_iter() {
let bitmap_bytes = CboRoaringBitmapCodec::bytes_encode(&bitmap).unwrap();
facet_exists_docids_writer.insert(fid.to_be_bytes(), &bitmap_bytes)?;
}
let facet_exists_docids_reader = writer_into_reader(facet_exists_docids_writer)?;
Ok((
sorter_into_reader(fid_docid_facet_numbers_sorter, indexer.clone())?,
sorter_into_reader(fid_docid_facet_strings_sorter, indexer.clone())?,
facet_exists_docids,
facet_exists_docids_reader,
))
}

29
milli/src/update/index_documents/extract/mod.rs
View File

@ -8,13 +8,12 @@ mod extract_word_docids;
mod extract_word_pair_proximity_docids;
mod extract_word_position_docids;
use std::collections::{BTreeMap, HashSet};
use std::collections::HashSet;
use std::fs::File;
use crossbeam_channel::Sender;
use log::debug;
use rayon::prelude::*;
use roaring::RoaringBitmap;
use self::extract_docid_word_positions::extract_docid_word_positions;
use self::extract_facet_number_docids::extract_facet_number_docids;
@ -73,21 +72,25 @@ pub(crate) fn data_from_obkv_documents(
let (
docid_word_positions_chunks,
(docid_fid_facet_numbers_chunks, (docid_fid_facet_strings_chunks, facet_exists_docids)),
(
docid_fid_facet_numbers_chunks,
(docid_fid_facet_strings_chunks, facet_exists_docids_chunks),
),
) = result?;
// merge facet_exists_docids hashmaps and send them as a typed chunk
// merge facet_exists_docids and send them as a typed chunk
{
let lmdb_writer_sx = lmdb_writer_sx.clone();
rayon::spawn(move || {
let mut all = BTreeMap::default();
for facet_exists_docids in facet_exists_docids {
for (field_id, docids) in facet_exists_docids {
let docids0 = all.entry(field_id).or_default();
*docids0 |= docids;
debug!("merge {} database", "facet-id-exists-docids");
match facet_exists_docids_chunks.merge(merge_cbo_roaring_bitmaps, &indexer) {
Ok(reader) => {
let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetExistsDocids(reader)));
}
Err(e) => {
let _ = lmdb_writer_sx.send(Err(e));
}
}
let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetExistsDocids(all)));
});
}
@ -228,7 +231,7 @@ fn send_and_extract_flattened_documents_data(
grenad::Reader<CursorClonableMmap>,
(
grenad::Reader<CursorClonableMmap>,
(grenad::Reader<CursorClonableMmap>, BTreeMap<FieldId, RoaringBitmap>),
(grenad::Reader<CursorClonableMmap>, grenad::Reader<File>),
),
)> {
let flattened_documents_chunk =
@ -273,7 +276,7 @@ fn send_and_extract_flattened_documents_data(
let (
docid_fid_facet_numbers_chunk,
docid_fid_facet_strings_chunk,
facet_exists_docids,
fid_facet_exists_docids_chunk,
) = extract_fid_docid_facet_values(
flattened_documents_chunk.clone(),
indexer.clone(),
@ -298,7 +301,7 @@ fn send_and_extract_flattened_documents_data(
Ok((
docid_fid_facet_numbers_chunk,
(docid_fid_facet_strings_chunk, facet_exists_docids),
(docid_fid_facet_strings_chunk, fid_facet_exists_docids_chunk),
))
},
);