snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-30 10:20:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Try to make facet indexing incremental

Browse Source
This commit is contained in:
Loïc Lecrenier
2022-09-01 08:17:27 +02:00
committed by Loïc Lecrenier
parent d30c89e345
commit 85824ee203
6 changed files with 55 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
milli/src/update/facet/bulk.rs
View File

@ -6,7 +6,7 @@ use crate::update::index_documents::{create_writer, write_into_lmdb_database, wr
use crate::{FieldId, Index, Result};
use grenad::CompressionType;
use heed::types::ByteSlice;
use heed::{BytesEncode, Error, RoTxn};
use heed::{BytesEncode, Error, RoTxn, RwTxn};
use log::debug;
use roaring::RoaringBitmap;
use std::cmp;
@ -21,12 +21,19 @@ pub struct FacetsUpdateBulk<'i> {
pub(crate) chunk_compression_level: Option<u32>,
level_group_size: usize,
min_level_size: usize,
put_faceted_docids_in_main: fn(&Index, &mut RwTxn, FieldId, &RoaringBitmap) -> heed::Result<()>,
}
impl<'i> FacetsUpdateBulk<'i> {
pub fn new(
index: &'i Index,
database: heed::Database<FacetKeyCodec<MyByteSlice>, FacetGroupValueCodec>,
put_faceted_docids_in_main: fn(
&Index,
&mut RwTxn,
FieldId,
&RoaringBitmap,
) -> heed::Result<()>,
) -> FacetsUpdateBulk<'i> {
FacetsUpdateBulk {
index,
@ -35,6 +42,7 @@ impl<'i> FacetsUpdateBulk<'i> {
chunk_compression_level: None,
level_group_size: 4,
min_level_size: 5,
put_faceted_docids_in_main,
}
}
@ -78,8 +86,12 @@ impl<'i> FacetsUpdateBulk<'i> {
let (level_readers, all_docids) =
self.compute_levels_for_field_id(field_id, &nested_wtxn)?;
// TODO: this will need to be an argument to Facets as well
self.index.put_string_faceted_documents_ids(&mut nested_wtxn, field_id, &all_docids)?;
(self.put_faceted_docids_in_main)(
&self.index,
&mut nested_wtxn,
field_id,
&all_docids,
)?;
for level_reader in level_readers {
// TODO: append instead of write with merge