snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-31 16:06:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

get back to what we were doingb efore in the DB cache and with the restricted field id

Browse Source
This commit is contained in:
Tamo
2024-05-15 18:00:39 +02:00
parent ad4d8502b3
commit 5542f1d9f1
2 changed files with 99 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
milli/src/search/new/db_cache.rs
View File

@@ -159,36 +159,58 @@ impl<'ctx> SearchContext<'ctx> {
/// Retrieve or insert the given value in the `word_docids` database. /// Retrieve or insert the given value in the `word_docids` database.
fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> { fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(word).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.tolerant.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(word).as_str();
let keys: Vec<_> =
restricted_fids.tolerant.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
word, word,
&keys[..], &keys[..],
&mut self.db_cache.word_docids, &mut self.db_cache.word_docids,
self.index.word_fid_docids.remap_data_type::<Bytes>(), self.index.word_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
word,
self.word_interner.get(word).as_str(),
&mut self.db_cache.word_docids,
self.index.word_docids.remap_data_type::<Bytes>(),
),
}
} }
fn get_db_exact_word_docids( fn get_db_exact_word_docids(
&mut self, &mut self,
word: Interned<String>, word: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(word).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.exact.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(word).as_str();
let keys: Vec<_> =
restricted_fids.exact.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
word, word,
&keys[..], &keys[..],
&mut self.db_cache.exact_word_docids, &mut self.db_cache.exact_word_docids,
self.index.word_fid_docids.remap_data_type::<Bytes>(), self.index.word_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
word,
self.word_interner.get(word).as_str(),
&mut self.db_cache.exact_word_docids,
self.index.exact_word_docids.remap_data_type::<Bytes>(),
),
}
} }
pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> { pub fn word_prefix_docids(&mut self, prefix: Word) -> Result<Option<RoaringBitmap>> {
@@ -216,36 +238,58 @@ impl<'ctx> SearchContext<'ctx> {
&mut self, &mut self,
prefix: Interned<String>, prefix: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(prefix).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.tolerant.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(prefix).as_str();
let keys: Vec<_> =
restricted_fids.tolerant.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
prefix, prefix,
&keys[..], &keys[..],
&mut self.db_cache.word_prefix_docids, &mut self.db_cache.word_prefix_docids,
self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(), self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
prefix,
self.word_interner.get(prefix).as_str(),
&mut self.db_cache.word_prefix_docids,
self.index.word_prefix_docids.remap_data_type::<Bytes>(),
),
}
} }
fn get_db_exact_word_prefix_docids( fn get_db_exact_word_prefix_docids(
&mut self, &mut self,
prefix: Interned<String>, prefix: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
let interned = self.word_interner.get(prefix).as_str(); match &self.restricted_fids {
let keys: Vec<_> = Some(restricted_fids) => {
self.searchable_fids.exact.iter().map(|(fid, _weight)| (interned, *fid)).collect(); let interned = self.word_interner.get(prefix).as_str();
let keys: Vec<_> =
restricted_fids.exact.iter().map(|(fid, _)| (interned, *fid)).collect();
DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
prefix, prefix,
&keys[..], &keys[..],
&mut self.db_cache.exact_word_prefix_docids, &mut self.db_cache.exact_word_prefix_docids,
self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(), self.index.word_prefix_fid_docids.remap_data_type::<Bytes>(),
merge_cbo_roaring_bitmaps, merge_cbo_roaring_bitmaps,
) )
}
None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn,
prefix,
self.word_interner.get(prefix).as_str(),
&mut self.db_cache.exact_word_prefix_docids,
self.index.exact_word_prefix_docids.remap_data_type::<Bytes>(),
),
}
} }
pub fn get_db_word_pair_proximity_docids( pub fn get_db_word_pair_proximity_docids(
@@ -421,8 +465,8 @@ impl<'ctx> SearchContext<'ctx> {
word: Interned<String>, word: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the list of searchable, return None. // if the requested fid isn't in the restricted list, return None.
if !self.searchable_fids.contains(&fid) { if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None); return Ok(None);
} }
@@ -440,8 +484,8 @@ impl<'ctx> SearchContext<'ctx> {
word_prefix: Interned<String>, word_prefix: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the searchable list, return None. // if the requested fid isn't in the restricted list, return None.
if !self.searchable_fids.contains(&fid) { if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None); return Ok(None);
} }

19
milli/src/search/new/mod.rs
View File

@@ -66,7 +66,7 @@ pub struct SearchContext<'ctx> {
pub phrase_interner: DedupInterner<Phrase>, pub phrase_interner: DedupInterner<Phrase>,
pub term_interner: Interner<QueryTerm>, pub term_interner: Interner<QueryTerm>,
pub phrase_docids: PhraseDocIdsCache, pub phrase_docids: PhraseDocIdsCache,
pub searchable_fids: SearchableFids, pub restricted_fids: Option<RestrictedFids>,
} }
impl<'ctx> SearchContext<'ctx> { impl<'ctx> SearchContext<'ctx> {
@@ -92,7 +92,7 @@ impl<'ctx> SearchContext<'ctx> {
phrase_interner: <_>::default(), phrase_interner: <_>::default(),
term_interner: <_>::default(), term_interner: <_>::default(),
phrase_docids: <_>::default(), phrase_docids: <_>::default(),
searchable_fids: SearchableFids { tolerant, exact }, restricted_fids: None,
}) })
} }
@@ -103,7 +103,7 @@ impl<'ctx> SearchContext<'ctx> {
let mut wildcard = false; let mut wildcard = false;
let mut restricted_fids = SearchableFids::default(); let mut restricted_fids = RestrictedFids::default();
for field_name in attributes_to_search_on { for field_name in attributes_to_search_on {
if field_name == "*" { if field_name == "*" {
wildcard = true; wildcard = true;
@@ -141,14 +141,9 @@ impl<'ctx> SearchContext<'ctx> {
} }
if wildcard { if wildcard {
let (exact, tolerant) = searchable_names self.restricted_fids = None;
.iter()
.map(|(_name, fid, weight)| (*fid, *weight))
.partition(|(fid, _weight)| exact_attributes_ids.contains(fid));
self.searchable_fids = SearchableFids { tolerant, exact };
} else { } else {
self.searchable_fids = restricted_fids; self.restricted_fids = Some(restricted_fids);
} }
Ok(()) Ok(())
@@ -171,12 +166,12 @@ impl Word {
} }
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct SearchableFids { pub struct RestrictedFids {
pub tolerant: Vec<(FieldId, Weight)>, pub tolerant: Vec<(FieldId, Weight)>,
pub exact: Vec<(FieldId, Weight)>, pub exact: Vec<(FieldId, Weight)>,
} }
impl SearchableFids { impl RestrictedFids {
pub fn contains(&self, fid: &FieldId) -> bool { pub fn contains(&self, fid: &FieldId) -> bool {
self.tolerant.iter().any(|(id, _)| id == fid) || self.exact.iter().any(|(id, _)| id == fid) self.tolerant.iter().any(|(id, _)| id == fid) || self.exact.iter().any(|(id, _)| id == fid)
} }