snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-25 13:06:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve documents deletion by iterating over all the word pair positions

Browse Source
This commit is contained in:
Kerollmops
2020-10-27 18:50:09 +01:00
parent 3889d956d9
commit d6338af766
1 changed files with 11 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/update/delete_documents.rs
View File

@@ -2,7 +2,6 @@ use std::borrow::Cow;
use std::convert::TryFrom;
use fst::{IntoStreamer, Streamer};
use itertools::Itertools;
use roaring::RoaringBitmap;
use crate::{Index, BEU32};
@@ -168,21 +167,17 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
// We write the new words FST into the main database.
self.index.put_words_fst(self.wtxn, &new_words_fst)?;
// We delete the documents ids that are under the pairs of words we found.
// TODO We can maybe improve this by using the `compute_words_pair_proximities`
// function instead of iterating over all the possible word pairs.
for ((w1, _), (w2, _)) in words.iter().cartesian_product(&words) {
let start = &(w1.as_str(), w2.as_str(), 0);
let end = &(w1.as_str(), w2.as_str(), 7);
let mut iter = word_pair_proximity_docids.range_mut(self.wtxn, &(start..=end))?;
while let Some(result) = iter.next() {
let ((w1, w2, prox), mut docids) = result?;
docids.difference_with(&documents_ids);
if docids.is_empty() {
iter.del_current()?;
} else {
iter.put_current(&(w1, w2, prox), &docids)?;
}
// We delete the documents ids that are under the pairs of words,
// it is faster and use no memory to iterate over all the words pairs than
// to compute the cartesian product of every words of the deleted documents.
let mut iter = word_pair_proximity_docids.iter_mut(self.wtxn)?;
while let Some(result) = iter.next() {
let ((w1, w2, prox), mut docids) = result?;
docids.difference_with(&documents_ids);
if docids.is_empty() {
iter.del_current()?;
} else {
iter.put_current(&(w1, w2, prox), &docids)?;
}
}