snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-28 09:11:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Prepare refactor of facets database

Browse Source 
Prepare refactor of facets database
This commit is contained in:
Loïc Lecrenier
2022-08-29 16:01:54 +02:00
committed by Loïc Lecrenier
parent 004c09a8e2
commit c3f49f766d
27 changed files with 1662 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
milli/src/search/criteria/asc_desc.rs
View File

@ -7,7 +7,7 @@ use roaring::RoaringBitmap;
use super::{Criterion, CriterionParameters, CriterionResult};
use crate::search::criteria::{resolve_query_tree, CriteriaBuilder};
use crate::search::facet::{FacetNumberIter, FacetStringIter};
// use crate::search::facet::FacetStringIter;
use crate::search::query_tree::Operation;
use crate::{FieldId, Index, Result};
@ -186,23 +186,24 @@ fn facet_ordered<'t>(
iterative_facet_string_ordered_iter(index, rtxn, field_id, is_ascending, candidates)?;
Ok(Box::new(number_iter.chain(string_iter).map(Ok)) as Box<dyn Iterator<Item = _>>)
} else {
let facet_number_fn = if is_ascending {
FacetNumberIter::new_reducing
} else {
FacetNumberIter::new_reverse_reducing
};
let number_iter = facet_number_fn(rtxn, index, field_id, candidates.clone())?
.map(|res| res.map(|(_, docids)| docids));
todo!()
// let facet_number_fn = if is_ascending {
// FacetNumberIter::new_reducing
// } else {
// FacetNumberIter::new_reverse_reducing
// };
// let number_iter = facet_number_fn(rtxn, index, field_id, candidates.clone())?
// .map(|res| res.map(|(_, docids)| docids));
let facet_string_fn = if is_ascending {
FacetStringIter::new_reducing
} else {
FacetStringIter::new_reverse_reducing
};
let string_iter = facet_string_fn(rtxn, index, field_id, candidates)?
.map(|res| res.map(|(_, _, docids)| docids));
// let facet_string_fn = if is_ascending {
// FacetStringIter::new_reducing
// } else {
// FacetStringIter::new_reverse_reducing
// };
// let string_iter = facet_string_fn(rtxn, index, field_id, candidates)?
// .map(|res| res.map(|(_, _, docids)| docids));
Ok(Box::new(number_iter.chain(string_iter)))
// Ok(Box::new(number_iter.chain(string_iter)))
}
}

10
milli/src/search/distinct/facet_distinct.rs
View File

@ -6,6 +6,7 @@ use roaring::RoaringBitmap;
use super::{Distinct, DocIter};
use crate::error::InternalError;
use crate::heed_codec::facet::new::FacetKey;
use crate::heed_codec::facet::*;
use crate::index::db_name;
use crate::{DocumentId, FieldId, Index, Result};
@ -47,13 +48,16 @@ impl<'a> FacetDistinctIter<'a> {
fn facet_string_docids(&self, key: &str) -> heed::Result<Option<RoaringBitmap>> {
self.index
.facet_id_string_docids
.get(self.txn, &(self.distinct, key))
.map(|result| result.map(|(_original, docids)| docids))
.get(self.txn, &FacetKey { field_id: self.distinct, level: 0, left_bound: key })
.map(|opt| opt.map(|v| v.bitmap))
}
fn facet_number_docids(&self, key: f64) -> heed::Result<Option<RoaringBitmap>> {
// get facet docids on level 0
self.index.facet_id_f64_docids.get(self.txn, &(self.distinct, 0, key, key))
self.index
.facet_id_f64_docids
.get(self.txn, &FacetKey { field_id: self.distinct, level: 0, left_bound: key })
.map(|opt| opt.map(|v| v.bitmap))
}
fn distinct_string(&mut self, id: DocumentId) -> Result<()> {

120
milli/src/search/facet/facet_distribution.rs
View File

@ -7,10 +7,8 @@ use roaring::RoaringBitmap;
use crate::error::UserError;
use crate::facet::FacetType;
use crate::heed_codec::facet::{
FacetStringLevelZeroCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec,
};
use crate::search::facet::{FacetNumberIter, FacetNumberRange, FacetStringIter};
use crate::heed_codec::facet::{FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec};
// use crate::search::facet::FacetStringIter;
use crate::{FieldId, Index, Result};
/// The default number of values by facets that will
@ -133,21 +131,22 @@ impl<'a> FacetDistribution<'a> {
candidates: &RoaringBitmap,
distribution: &mut BTreeMap<String, u64>,
) -> heed::Result<()> {
let iter =
FacetNumberIter::new_non_reducing(self.rtxn, self.index, field_id, candidates.clone())?;
todo!()
// let iter =
// FacetNumberIter::new_non_reducing(self.rtxn, self.index, field_id, candidates.clone())?;
for result in iter {
let (value, mut docids) = result?;
docids &= candidates;
if !docids.is_empty() {
distribution.insert(value.to_string(), docids.len());
}
if distribution.len() == self.max_values_per_facet {
break;
}
}
// for result in iter {
// let (value, mut docids) = result?;
// docids &= candidates;
// if !docids.is_empty() {
// distribution.insert(value.to_string(), docids.len());
// }
// if distribution.len() == self.max_values_per_facet {
// break;
// }
// }
Ok(())
// Ok(())
}
fn facet_strings_distribution_from_facet_levels(
@ -156,21 +155,22 @@ impl<'a> FacetDistribution<'a> {
candidates: &RoaringBitmap,
distribution: &mut BTreeMap<String, u64>,
) -> heed::Result<()> {
let iter =
FacetStringIter::new_non_reducing(self.rtxn, self.index, field_id, candidates.clone())?;
todo!()
// let iter =
// FacetStringIter::new_non_reducing(self.rtxn, self.index, field_id, candidates.clone())?;
for result in iter {
let (_normalized, original, mut docids) = result?;
docids &= candidates;
if !docids.is_empty() {
distribution.insert(original.to_string(), docids.len());
}
if distribution.len() == self.max_values_per_facet {
break;
}
}
// for result in iter {
// let (_normalized, original, mut docids) = result?;
// docids &= candidates;
// if !docids.is_empty() {
// distribution.insert(original.to_string(), docids.len());
// }
// if distribution.len() == self.max_values_per_facet {
// break;
// }
// }
Ok(())
// Ok(())
}
/// Placeholder search, a.k.a. no candidates were specified. We iterate throught the
@ -179,41 +179,43 @@ impl<'a> FacetDistribution<'a> {
&self,
field_id: FieldId,
) -> heed::Result<BTreeMap<String, u64>> {
let mut distribution = BTreeMap::new();
todo!()
// let mut distribution = BTreeMap::new();
let db = self.index.facet_id_f64_docids;
let range = FacetNumberRange::new(self.rtxn, db, field_id, 0, Unbounded, Unbounded)?;
// let db = self.index.facet_id_f64_docids;
// let range = FacetNumberRange::new(self.rtxn, db, field_id, 0, Unbounded, Unbounded)?;
for result in range {
let ((_, _, value, _), docids) = result?;
distribution.insert(value.to_string(), docids.len());
if distribution.len() == self.max_values_per_facet {
break;
}
}
// for result in range {
// let ((_, _, value, _), docids) = result?;
// distribution.insert(value.to_string(), docids.len());
// if distribution.len() == self.max_values_per_facet {
// break;
// }
// }
let iter = self
.index
.facet_id_string_docids
.remap_key_type::<ByteSlice>()
.prefix_iter(self.rtxn, &field_id.to_be_bytes())?
.remap_key_type::<FacetStringLevelZeroCodec>();
// let iter = self
// .index
// .facet_id_string_docids
// .remap_key_type::<ByteSlice>()
// .prefix_iter(self.rtxn, &field_id.to_be_bytes())?
// .remap_key_type::<FacetStringLevelZeroCodec>();
let mut normalized_distribution = BTreeMap::new();
for result in iter {
let ((_, normalized_value), (original_value, docids)) = result?;
normalized_distribution.insert(normalized_value, (original_value, docids.len()));
if normalized_distribution.len() == self.max_values_per_facet {
break;
}
}
// let mut normalized_distribution = BTreeMap::new();
// for result in iter {
// let ((_, normalized_value), group_value) = result?;
// normalized_distribution
// .insert(normalized_value, (normalized_value, group_value.bitmap.len()));
// if normalized_distribution.len() == self.max_values_per_facet {
// break;
// }
// }
let iter = normalized_distribution
.into_iter()
.map(|(_normalized, (original, count))| (original.to_string(), count));
distribution.extend(iter);
// let iter = normalized_distribution
// .into_iter()
// .map(|(_normalized, (original, count))| (original.to_string(), count));
// distribution.extend(iter);
Ok(distribution)
// Ok(distribution)
}
fn facet_values(&self, field_id: FieldId) -> heed::Result<BTreeMap<String, u64>> {

539
milli/src/search/facet/facet_number.rs
View File

@ -1,248 +1,335 @@
use std::ops::Bound::{self, Excluded, Included, Unbounded};
// use std::ops::Bound::{self, Excluded, Included, Unbounded};
use either::Either::{self, Left, Right};
use heed::types::{ByteSlice, DecodeIgnore};
use heed::{Database, LazyDecode, RoRange, RoRevRange};
use roaring::RoaringBitmap;
// use either::Either::{self, Left, Right};
// use heed::types::{ByteSlice, DecodeIgnore};
// use heed::{BytesDecode, BytesEncode, Database, Lazy, LazyDecode, RoRange, RoRevRange};
// use obkv::Key;
// use roaring::RoaringBitmap;
use crate::heed_codec::facet::FacetLevelValueF64Codec;
use crate::heed_codec::CboRoaringBitmapCodec;
use crate::{FieldId, Index};
// use crate::heed_codec::facet::new::ordered_f64_codec::OrderedF64Codec;
// use crate::heed_codec::facet::new::{FacetGroupValueCodec, FacetKey, FacetKeyCodec};
// use crate::heed_codec::CboRoaringBitmapCodec;
// use crate::{FieldId, Index};
pub struct FacetNumberRange<'t> {
iter: RoRange<'t, FacetLevelValueF64Codec, LazyDecode<CboRoaringBitmapCodec>>,
end: Bound<f64>,
}
// pub struct FacetNumberRange<'t, 'e> {
// rtxn: &'t heed::RoTxn<'e>,
// db: Database<FacetKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
// iter: RoRange<'t, FacetKeyCodec<OrderedF64Codec>, LazyDecode<FacetGroupValueCodec>>,
// max_bound: f64,
// previous: Option<(FacetKey<f64>, Lazy<'t, FacetGroupValueCodec>)>,
// field_id: FieldId,
// end: Bound<f64>,
// }
impl<'t> FacetNumberRange<'t> {
pub fn new(
rtxn: &'t heed::RoTxn,
db: Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>,
field_id: FieldId,
level: u8,
left: Bound<f64>,
right: Bound<f64>,
) -> heed::Result<FacetNumberRange<'t>> {
let left_bound = match left {
Included(left) => Included((field_id, level, left, f64::MIN)),
Excluded(left) => Excluded((field_id, level, left, f64::MIN)),
Unbounded => Included((field_id, level, f64::MIN, f64::MIN)),
};
let right_bound = Included((field_id, level, f64::MAX, f64::MAX));
let iter = db.lazily_decode_data().range(rtxn, &(left_bound, right_bound))?;
Ok(FacetNumberRange { iter, end: right })
}
}
// impl<'t, 'e> FacetNumberRange<'t, 'e> {
// pub fn new(
// rtxn: &'t heed::RoTxn<'e>,
// db: Database<FacetKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
// field_id: FieldId,
// level: u8,
// left: Bound<f64>,
// right: Bound<f64>,
// ) -> heed::Result<FacetNumberRange<'t, 'e>> {
// let left_bound = match left {
// Included(left_bound) => Included(FacetKey { field_id, level, left_bound }),
// Excluded(left_bound) => Excluded(FacetKey { field_id, level, left_bound }),
// Unbounded => Included(FacetKey { field_id, level, left_bound: f64::MIN }),
// };
impl<'t> Iterator for FacetNumberRange<'t> {
type Item = heed::Result<((FieldId, u8, f64, f64), RoaringBitmap)>;
// let mut iter = db.lazily_decode_data().range(rtxn, &(left_bound, Unbounded))?;
// let mut previous = iter.next().transpose()?;
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
Some(Ok(((fid, level, left, right), docids))) => {
let must_be_returned = match self.end {
Included(end) => right <= end,
Excluded(end) => right < end,
Unbounded => true,
};
if must_be_returned {
match docids.decode() {
Ok(docids) => Some(Ok(((fid, level, left, right), docids))),
Err(e) => Some(Err(e)),
}
} else {
None
}
}
Some(Err(e)) => Some(Err(e)),
None => None,
}
}
}
// // Compute the maximum end bound by looking at the key of the last element in level 0
// let mut prefix_level_0 = vec![];
// prefix_level_0.extend_from_slice(&field_id.to_be_bytes());
// prefix_level_0.push(level);
pub struct FacetNumberRevRange<'t> {
iter: RoRevRange<'t, FacetLevelValueF64Codec, LazyDecode<CboRoaringBitmapCodec>>,
end: Bound<f64>,
}
// let mut rev_iter =
// db.as_polymorph().rev_prefix_iter::<_, ByteSlice, ByteSlice>(rtxn, &prefix_level_0)?;
impl<'t> FacetNumberRevRange<'t> {
pub fn new(
rtxn: &'t heed::RoTxn,
db: Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>,
field_id: FieldId,
level: u8,
left: Bound<f64>,
right: Bound<f64>,
) -> heed::Result<FacetNumberRevRange<'t>> {
let left_bound = match left {
Included(left) => Included((field_id, level, left, f64::MIN)),
Excluded(left) => Excluded((field_id, level, left, f64::MIN)),
Unbounded => Included((field_id, level, f64::MIN, f64::MIN)),
};
let right_bound = Included((field_id, level, f64::MAX, f64::MAX));
let iter = db.lazily_decode_data().rev_range(rtxn, &(left_bound, right_bound))?;
Ok(FacetNumberRevRange { iter, end: right })
}
}
// let rev_iter_first = rev_iter.next().transpose()?;
// let max_bound = if let Some((max_bound_key, _)) = rev_iter_first {
// let max_bound_key =
// FacetKeyCodec::<OrderedF64Codec>::bytes_decode(max_bound_key).unwrap();
// max_bound_key.left_bound
// } else {
// // I can't imagine when that would happen, but let's handle it correctly anyway
// // by making the iterator empty
// previous = None;
// 0.0 // doesn't matter since previous = None so the iterator will always early exit
// // and return None itself
// };
impl<'t> Iterator for FacetNumberRevRange<'t> {
type Item = heed::Result<((FieldId, u8, f64, f64), RoaringBitmap)>;
// Ok(FacetNumberRange { rtxn, db, iter, field_id, previous, max_bound, end: right })
// }
// }
fn next(&mut self) -> Option<Self::Item> {
loop {
match self.iter.next() {
Some(Ok(((fid, level, left, right), docids))) => {
let must_be_returned = match self.end {
Included(end) => right <= end,
Excluded(end) => right < end,
Unbounded => true,
};
if must_be_returned {
match docids.decode() {
Ok(docids) => return Some(Ok(((fid, level, left, right), docids))),
Err(e) => return Some(Err(e)),
}
}
continue;
}
Some(Err(e)) => return Some(Err(e)),
None => return None,
}
}
}
}
// impl<'t, 'e> Iterator for FacetNumberRange<'t, 'e> {
// type Item = heed::Result<(FacetKey<f64>, RoaringBitmap)>;
pub struct FacetNumberIter<'t> {
rtxn: &'t heed::RoTxn<'t>,
db: Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>,
field_id: FieldId,
level_iters: Vec<(RoaringBitmap, Either<FacetNumberRange<'t>, FacetNumberRevRange<'t>>)>,
must_reduce: bool,
}
// fn next(&mut self) -> Option<Self::Item> {
// // The idea here is to return the **previous** element only if the left
// // bound of the current key fits within the range given to the iter
// // if it doesn't, then there is still a chance that it must be returned,
// // but we need to check the actual right bound of the group by looking for
// // the key preceding the first key of the next group in level 0
impl<'t> FacetNumberIter<'t> {
/// Create a `FacetNumberIter` that will iterate on the different facet entries
/// (facet value + documents ids) and that will reduce the given documents ids
/// while iterating on the different facet levels.
pub fn new_reducing(
rtxn: &'t heed::RoTxn,
index: &'t Index,
field_id: FieldId,
documents_ids: RoaringBitmap,
) -> heed::Result<FacetNumberIter<'t>> {
let db = index.facet_id_f64_docids.remap_key_type::<FacetLevelValueF64Codec>();
let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
let highest_iter =
FacetNumberRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
let level_iters = vec![(documents_ids, Left(highest_iter))];
Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: true })
}
// let (prev_key, prev_value) = self.previous?;
/// Create a `FacetNumberIter` that will iterate on the different facet entries in reverse
/// (facet value + documents ids) and that will reduce the given documents ids
/// while iterating on the different facet levels.
pub fn new_reverse_reducing(
rtxn: &'t heed::RoTxn,
index: &'t Index,
field_id: FieldId,
documents_ids: RoaringBitmap,
) -> heed::Result<FacetNumberIter<'t>> {
let db = index.facet_id_f64_docids;
let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
let highest_iter =
FacetNumberRevRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
let level_iters = vec![(documents_ids, Right(highest_iter))];
Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: true })
}
// let (next_left_bound, next_previous) = if let Some(next) = self.iter.next() {
// let (key, group_value) = match next {
// Ok(n) => n,
// Err(e) => return Some(Err(e)),
// };
// (key.left_bound, Some((key, group_value)))
// } else {
// // we're at the end of the level iter, so we need to fetch the max bound instead
// (self.max_bound, None)
// };
// let must_be_returned = match self.end {
// Included(end) => next_left_bound <= end,
// Excluded(end) => next_left_bound < end,
// Unbounded => true,
// };
// if must_be_returned {
// match prev_value.decode() {
// Ok(group_value) => {
// self.previous = next_previous;
// Some(Ok((prev_key, group_value.bitmap)))
// }
// Err(e) => Some(Err(e)),
// }
// } else {
// // it still possible that we want to return the value (one last time)
// // but to do so, we need to fetch the right bound of the current group
// // this is done by getting the first element at level 0 of the next group
// // then iterating in reverse from it
// // once we have the right bound, we can compare it, and then return or not
// // then we still set self.previous to None so that no other element can return
// // from it?
// let mut level_0_key_prefix = vec![];
// level_0_key_prefix.extend_from_slice(&self.field_id.to_be_bytes());
// level_0_key_prefix.push(0);
// let key =
// FacetKey::<f64> { field_id: self.field_id, level: 0, left_bound: next_left_bound };
// let key_bytes = FacetKeyCodec::<OrderedF64Codec>::bytes_encode(&key).unwrap();
// level_0_key_prefix.extend_from_slice(&key_bytes);
/// Create a `FacetNumberIter` that will iterate on the different facet entries
/// (facet value + documents ids) and that will not reduce the given documents ids
/// while iterating on the different facet levels, possibly returning multiple times
/// a document id associated with multiple facet values.
pub fn new_non_reducing(
rtxn: &'t heed::RoTxn,
index: &'t Index,
field_id: FieldId,
documents_ids: RoaringBitmap,
) -> heed::Result<FacetNumberIter<'t>> {
let db = index.facet_id_f64_docids.remap_key_type::<FacetLevelValueF64Codec>();
let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
let highest_iter =
FacetNumberRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
let level_iters = vec![(documents_ids, Left(highest_iter))];
Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: false })
}
// let mut rev_iter_next_group_level_0 = self
// .db
// .as_polymorph()
// .rev_prefix_iter::<_, ByteSlice, ByteSlice>(&self.rtxn, &level_0_key_prefix)
// .unwrap();
// let (key_for_right_bound, _) = rev_iter_next_group_level_0.next().unwrap().unwrap();
// let key_for_right_bound =
// FacetKeyCodec::<OrderedF64Codec>::bytes_decode(key_for_right_bound).unwrap();
// let right_bound = key_for_right_bound.left_bound;
// let must_be_returned = match self.end {
// Included(end) => right_bound <= end,
// Excluded(end) => right_bound < end,
// Unbounded => unreachable!(),
// };
// self.previous = None;
// if must_be_returned {
// match prev_value.decode() {
// Ok(group_value) => Some(Ok((prev_key, group_value.bitmap))),
// Err(e) => Some(Err(e)),
// }
// } else {
// None
// }
// }
// }
// }
fn highest_level<X>(
rtxn: &'t heed::RoTxn,
db: Database<FacetLevelValueF64Codec, X>,
fid: FieldId,
) -> heed::Result<Option<u8>> {
let level = db
.remap_types::<ByteSlice, DecodeIgnore>()
.prefix_iter(rtxn, &fid.to_be_bytes())?
.remap_key_type::<FacetLevelValueF64Codec>()
.last()
.transpose()?
.map(|((_, level, _, _), _)| level);
Ok(level)
}
}
// pub struct FacetNumberRevRange<'t> {
// iter: RoRevRange<'t, FacetKeyCodec<OrderedF64Codec>, LazyDecode<FacetGroupValueCodec>>,
// end: Bound<f64>,
// }
impl<'t> Iterator for FacetNumberIter<'t> {
type Item = heed::Result<(f64, RoaringBitmap)>;
// impl<'t> FacetNumberRevRange<'t> {
// pub fn new(
// rtxn: &'t heed::RoTxn,
// db: Database<FacetKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
// field_id: FieldId,
// level: u8,
// left: Bound<f64>,
// right: Bound<f64>,
// ) -> heed::Result<FacetNumberRevRange<'t>> {
// let left_bound = match left {
// Included(left) => Included(FacetKey { field_id, level, left_bound: left }),
// Excluded(left) => Excluded(FacetKey { field_id, level, left_bound: left }),
// Unbounded => Included(FacetKey { field_id, level, left_bound: f64::MIN }),
// };
// let right_bound = Included(FacetKey { field_id, level, left_bound: f64::MAX });
// let iter = db.lazily_decode_data().rev_range(rtxn, &(left_bound, right_bound))?;
// Ok(FacetNumberRevRange { iter, end: right })
// }
// }
fn next(&mut self) -> Option<Self::Item> {
'outer: loop {
let (documents_ids, last) = self.level_iters.last_mut()?;
let is_ascending = last.is_left();
for result in last {
// If the last iterator must find an empty set of documents it means
// that we found all the documents in the sub level iterations already,
// we can pop this level iterator.
if documents_ids.is_empty() {
break;
}
// impl<'t> Iterator for FacetNumberRevRange<'t> {
// type Item = heed::Result<(FacetKey<f64>, RoaringBitmap)>;
match result {
Ok(((_fid, level, left, right), mut docids)) => {
docids &= &*documents_ids;
if !docids.is_empty() {
if self.must_reduce {
*documents_ids -= &docids;
}
// fn next(&mut self) -> Option<Self::Item> {
// loop {
// match self.iter.next() {
// Some(Ok((FacetKey { field_id, level, left_bound }, docids))) => {
// let must_be_returned = match self.end {
// Included(end) => todo!(), //right <= end,
// Excluded(end) => todo!(), //right < end,
// Unbounded => true,
// };
// if must_be_returned {
// match docids.decode() {
// Ok(docids) => {
// return Some(Ok((
// FacetKey { field_id, level, left_bound },
// docids.bitmap,
// )))
// }
// Err(e) => return Some(Err(e)),
// }
// }
// continue;
// }
// Some(Err(e)) => return Some(Err(e)),
// None => return None,
// }
// }
// }
// }
if level == 0 {
return Some(Ok((left, docids)));
}
// pub struct FacetNumberIter<'t, 'e> {
// rtxn: &'t heed::RoTxn<'t>,
// db: Database<FacetKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
// field_id: FieldId,
// level_iters: Vec<(RoaringBitmap, Either<FacetNumberRange<'t, 'e>, FacetNumberRevRange<'t>>)>,
// must_reduce: bool,
// }
let rtxn = self.rtxn;
let db = self.db;
let fid = self.field_id;
let left = Included(left);
let right = Included(right);
// impl<'t, 'e> FacetNumberIter<'t, 'e> {
// /// Create a `FacetNumberIter` that will iterate on the different facet entries
// /// (facet value + documents ids) and that will reduce the given documents ids
// /// while iterating on the different facet levels.
// pub fn new_reducing(
// rtxn: &'t heed::RoTxn<'e>,
// index: &'t Index,
// field_id: FieldId,
// documents_ids: RoaringBitmap,
// ) -> heed::Result<FacetNumberIter<'t, 'e>> {
// let db = index.facet_id_f64_docids;
// let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
// let highest_iter =
// FacetNumberRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
// let level_iters = vec![(documents_ids, Left(highest_iter))];
// Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: true })
// }
let result = if is_ascending {
FacetNumberRange::new(rtxn, db, fid, level - 1, left, right)
.map(Left)
} else {
FacetNumberRevRange::new(rtxn, db, fid, level - 1, left, right)
.map(Right)
};
// /// Create a `FacetNumberIter` that will iterate on the different facet entries in reverse
// /// (facet value + documents ids) and that will reduce the given documents ids
// /// while iterating on the different facet levels.
// pub fn new_reverse_reducing(
// rtxn: &'t heed::RoTxn<'e>,
// index: &'t Index,
// field_id: FieldId,
// documents_ids: RoaringBitmap,
// ) -> heed::Result<FacetNumberIter<'t, 'e>> {
// let db = index.facet_id_f64_docids;
// let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
// let highest_iter =
// FacetNumberRevRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
// let level_iters = vec![(documents_ids, Right(highest_iter))];
// Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: true })
// }
match result {
Ok(iter) => {
self.level_iters.push((docids, iter));
continue 'outer;
}
Err(e) => return Some(Err(e)),
}
}
}
Err(e) => return Some(Err(e)),
}
}
self.level_iters.pop();
}
}
}
// /// Create a `FacetNumberIter` that will iterate on the different facet entries
// /// (facet value + documents ids) and that will not reduce the given documents ids
// /// while iterating on the different facet levels, possibly returning multiple times
// /// a document id associated with multiple facet values.
// pub fn new_non_reducing(
// rtxn: &'t heed::RoTxn<'e>,
// index: &'t Index,
// field_id: FieldId,
// documents_ids: RoaringBitmap,
// ) -> heed::Result<FacetNumberIter<'t, 'e>> {
// let db = index.facet_id_f64_docids;
// let highest_level = Self::highest_level(rtxn, db, field_id)?.unwrap_or(0);
// let highest_iter =
// FacetNumberRange::new(rtxn, db, field_id, highest_level, Unbounded, Unbounded)?;
// let level_iters = vec![(documents_ids, Left(highest_iter))];
// Ok(FacetNumberIter { rtxn, db, field_id, level_iters, must_reduce: false })
// }
// fn highest_level<X>(
// rtxn: &'t heed::RoTxn,
// db: Database<FacetKeyCodec<OrderedF64Codec>, X>,
// fid: FieldId,
// ) -> heed::Result<Option<u8>> {
// let level = db
// .remap_types::<ByteSlice, DecodeIgnore>()
// .prefix_iter(rtxn, &fid.to_be_bytes())?
// .remap_key_type::<FacetKeyCodec<OrderedF64Codec>>()
// .last()
// .transpose()?
// .map(|(key, _)| key.level);
// Ok(level)
// }
// }
// impl<'t, 'e> Iterator for FacetNumberIter<'t, 'e> {
// type Item = heed::Result<(f64, RoaringBitmap)>;
// fn next(&mut self) -> Option<Self::Item> {
// 'outer: loop {
// let (documents_ids, last) = self.level_iters.last_mut()?;
// let is_ascending = last.is_left();
// for result in last {
// // If the last iterator must find an empty set of documents it means
// // that we found all the documents in the sub level iterations already,
// // we can pop this level iterator.
// if documents_ids.is_empty() {
// break;
// }
// match result {
// Ok((key, mut docids)) => {
// docids &= &*documents_ids;
// if !docids.is_empty() {
// if self.must_reduce {
// *documents_ids -= &docids;
// }
// if level == 0 {
// return Some(Ok((left, docids)));
// }
// let rtxn = self.rtxn;
// let db = self.db;
// let fid = self.field_id;
// let left = Included(left);
// let right = Included(right);
// let result = if is_ascending {
// FacetNumberRange::new(rtxn, db, fid, level - 1, left, right)
// .map(Left)
// } else {
// FacetNumberRevRange::new(rtxn, db, fid, level - 1, left, right)
// .map(Right)
// };
// match result {
// Ok(iter) => {
// self.level_iters.push((docids, iter));
// continue 'outer;
// }
// Err(e) => return Some(Err(e)),
// }
// }
// }
// Err(e) => return Some(Err(e)),
// }
// }
// self.level_iters.pop();
// }
// }
// }

1217
milli/src/search/facet/facet_string.rs
View File

File diff suppressed because it is too large Load Diff

268
milli/src/search/facet/filter.rs
View File

@ -1,16 +1,20 @@
use std::collections::HashSet;
use std::fmt::{Debug, Display};
use std::ops::Bound::{self, Excluded, Included};
use std::ops::RangeBounds;
use either::Either;
pub use filter_parser::{Condition, Error as FPError, FilterCondition, Span, Token};
use heed::types::DecodeIgnore;
use heed::LazyDecode;
use log::debug;
use roaring::RoaringBitmap;
use super::FacetNumberRange;
// use super::FacetNumberRange;
use crate::error::{Error, UserError};
use crate::heed_codec::facet::FacetLevelValueF64Codec;
use crate::heed_codec::facet::new::ordered_f64_codec::OrderedF64Codec;
use crate::heed_codec::facet::new::{FacetGroupValueCodec, FacetKey, FacetKeyCodec};
// use crate::heed_codec::facet::FacetLevelValueF64Codec;
use crate::{
distance_between_two_points, lat_lng_to_xyz, CboRoaringBitmapCodec, FieldId, Index, Result,
};
@ -144,18 +148,29 @@ impl<'a> Filter<'a> {
}
}
fn explore_facet_number_levels(
rtxn: &heed::RoTxn,
db: heed::Database<FacetKeyCodec<OrderedF64Codec>, FacetGroupValueCodec>,
field_id: FieldId,
) {
}
impl<'a> Filter<'a> {
/// Aggregates the documents ids that are part of the specified range automatically
/// going deeper through the levels.
fn explore_facet_number_levels(
rtxn: &heed::RoTxn,
db: heed::Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>,
db: heed::Database<FacetKeyCodec<OrderedF64Codec>, CboRoaringBitmapCodec>,
field_id: FieldId,
level: u8,
left: Bound<f64>,
right: Bound<f64>,
output: &mut RoaringBitmap,
) -> Result<()> {
// level must be > 0, I'll create a separate function for level 0
// if level == 0 {
// call that function
//}
match (left, right) {
// If the request is an exact value we must go directly to the deepest level.
(Included(l), Included(r)) if l == r && level > 0 => {
@ -170,87 +185,121 @@ impl<'a> Filter<'a> {
(Excluded(l), Included(r)) if l >= r => return Ok(()),
(_, _) => (),
}
let mut left_found = None;
let mut right_found = None;
// We must create a custom iterator to be able to iterate over the
// requested range as the range iterator cannot express some conditions.
let iter = FacetNumberRange::new(rtxn, db, field_id, level, left, right)?;
debug!("Iterating between {:?} and {:?} (level {})", left, right, level);
for (i, result) in iter.enumerate() {
let ((_fid, level, l, r), docids) = result?;
debug!("{:?} to {:?} (level {}) found {} documents", l, r, level, docids.len());
*output |= docids;
// We save the leftest and rightest bounds we actually found at this level.
if i == 0 {
left_found = Some(l);
}
right_found = Some(r);
}
// Can we go deeper?
let deeper_level = match level.checked_sub(1) {
Some(level) => level,
None => return Ok(()),
let range_start_key = FacetKey {
field_id,
level,
left_bound: match left {
Included(l) => l,
Excluded(l) => l,
Bound::Unbounded => f64::MIN,
},
};
let mut range_iter = db
.remap_data_type::<LazyDecode<FacetGroupValueCodec>>()
.range(rtxn, &(range_start_key..))?;
// We must refine the left and right bounds of this range by retrieving the
// missing part in a deeper level.
match left_found.zip(right_found) {
Some((left_found, right_found)) => {
// If the bound is satisfied we avoid calling this function again.
if !matches!(left, Included(l) if l == left_found) {
let sub_right = Excluded(left_found);
debug!(
"calling left with {:?} to {:?} (level {})",
left, sub_right, deeper_level
);
Self::explore_facet_number_levels(
rtxn,
db,
field_id,
deeper_level,
left,
sub_right,
output,
)?;
}
if !matches!(right, Included(r) if r == right_found) {
let sub_left = Excluded(right_found);
debug!(
"calling right with {:?} to {:?} (level {})",
sub_left, right, deeper_level
);
Self::explore_facet_number_levels(
rtxn,
db,
field_id,
deeper_level,
sub_left,
right,
output,
)?;
}
}
None => {
// If we found nothing at this level it means that we must find
// the same bounds but at a deeper, more precise level.
Self::explore_facet_number_levels(
rtxn,
db,
field_id,
deeper_level,
left,
right,
output,
)?;
}
let (mut previous_facet_key, mut previous_value) = range_iter.next().unwrap()?;
while let Some(el) = range_iter.next() {
let (facet_key, value) = el?;
let range = (Included(previous_facet_key.left_bound), Excluded(facet_key.left_bound));
// if the current range intersects with the query range, then go deeper
// what does it mean for two ranges to intersect?
let gte_left = match left {
Included(l) => previous_facet_key.left_bound >= l,
Excluded(l) => previous_facet_key.left_bound > l, // TODO: not true?
Bound::Unbounded => true,
};
let lte_right = match right {
Included(r) => facet_key.left_bound <= r,
Excluded(r) => facet_key.left_bound < r,
Bound::Unbounded => true,
};
}
// at this point, previous_facet_key and previous_value are the last groups in the level
// we must also check whether we should visit this group
Ok(())
todo!();
// let mut left_found = None;
// let mut right_found = None;
// // We must create a custom iterator to be able to iterate over the
// // requested range as the range iterator cannot express some conditions.
// let iter = FacetNumberRange::new(rtxn, db, field_id, level, left, right)?;
// debug!("Iterating between {:?} and {:?} (level {})", left, right, level);
// for (i, result) in iter.enumerate() {
// let ((_fid, level, l, r), docids) = result?;
// debug!("{:?} to {:?} (level {}) found {} documents", l, r, level, docids.len());
// *output |= docids;
// // We save the leftest and rightest bounds we actually found at this level.
// if i == 0 {
// left_found = Some(l);
// }
// right_found = Some(r);
// }
// // Can we go deeper?
// let deeper_level = match level.checked_sub(1) {
// Some(level) => level,
// None => return Ok(()),
// };
// // We must refine the left and right bounds of this range by retrieving the
// // missing part in a deeper level.
// match left_found.zip(right_found) {
// Some((left_found, right_found)) => {
// // If the bound is satisfied we avoid calling this function again.
// if !matches!(left, Included(l) if l == left_found) {
// let sub_right = Excluded(left_found);
// debug!(
// "calling left with {:?} to {:?} (level {})",
// left, sub_right, deeper_level
// );
// Self::explore_facet_number_levels(
// rtxn,
// db,
// field_id,
// deeper_level,
// left,
// sub_right,
// output,
// )?;
// }
// if !matches!(right, Included(r) if r == right_found) {
// let sub_left = Excluded(right_found);
// debug!(
// "calling right with {:?} to {:?} (level {})",
// sub_left, right, deeper_level
// );
// Self::explore_facet_number_levels(
// rtxn,
// db,
// field_id,
// deeper_level,
// sub_left,
// right,
// output,
// )?;
// }
// }
// None => {
// // If we found nothing at this level it means that we must find
// // the same bounds but at a deeper, more precise level.
// Self::explore_facet_number_levels(
// rtxn,
// db,
// field_id,
// deeper_level,
// left,
// right,
// output,
// )?;
// }
// }
// Ok(())
}
fn evaluate_operator(
@ -277,23 +326,27 @@ impl<'a> Filter<'a> {
return Ok(exist);
}
Condition::Equal(val) => {
let (_original_value, string_docids) = strings_db
.get(rtxn, &(field_id, &val.value().to_lowercase()))?
let string_docids = strings_db
.get(
rtxn,
&FacetKey { field_id, level: 0, left_bound: &val.value().to_lowercase() },
)?
.map(|v| v.bitmap)
.unwrap_or_default();
let number = val.parse::<f64>().ok();
let number_docids = match number {
Some(n) => {
let n = Included(n);
let mut output = RoaringBitmap::new();
Self::explore_facet_number_levels(
rtxn,
numbers_db,
field_id,
0,
n,
n,
&mut output,
)?;
// Self::explore_facet_number_levels(
// rtxn,
// numbers_db,
// field_id,
// 0,
// n,
// n,
// &mut output,
// )?;
output
}
None => RoaringBitmap::new(),
@ -312,21 +365,32 @@ impl<'a> Filter<'a> {
// that's fine if it don't, the value just before will be returned instead.
let biggest_level = numbers_db
.remap_data_type::<DecodeIgnore>()
.get_lower_than_or_equal_to(rtxn, &(field_id, u8::MAX, f64::MAX, f64::MAX))?
.and_then(|((id, level, _, _), _)| if id == field_id { Some(level) } else { None });
.get_lower_than_or_equal_to(
rtxn,
&FacetKey { field_id, level: u8::MAX, left_bound: f64::MAX },
)?
.and_then(
|(FacetKey { field_id: id, level, .. }, _)| {
if id == field_id {
Some(level)
} else {
None
}
},
);
match biggest_level {
Some(level) => {
let mut output = RoaringBitmap::new();
Self::explore_facet_number_levels(
rtxn,
numbers_db,
field_id,
level,
left,
right,
&mut output,
)?;
// Self::explore_facet_number_levels(
// rtxn,
// numbers_db,
// field_id,
// level,
// left,
// right,
// &mut output,
// )?;
Ok(output)
}
None => Ok(RoaringBitmap::new()),

4
milli/src/search/facet/mod.rs
View File

@ -1,6 +1,6 @@
pub use self::facet_distribution::{FacetDistribution, DEFAULT_VALUES_PER_FACET};
pub use self::facet_number::{FacetNumberIter, FacetNumberRange, FacetNumberRevRange};
pub use self::facet_string::FacetStringIter;
// pub use self::facet_number::{FacetNumberIter, FacetNumberRange, FacetNumberRevRange};
// pub use self::facet_string::FacetStringIter;
pub use self::filter::Filter;
mod facet_distribution;

2
milli/src/search/mod.rs
View File

@ -15,7 +15,7 @@ use log::debug;
use once_cell::sync::Lazy;
use roaring::bitmap::RoaringBitmap;
pub use self::facet::{FacetDistribution, FacetNumberIter, Filter, DEFAULT_VALUES_PER_FACET};
pub use self::facet::{FacetDistribution, /* FacetNumberIter,*/ Filter, DEFAULT_VALUES_PER_FACET,};
use self::fst_utils::{Complement, Intersection, StartsWith, Union};
pub use self::matches::{
FormatOptions, MatchBounds, Matcher, MatcherBuilder, MatchingWord, MatchingWords,