mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-25 07:41:00 +00:00
Refactor facet database tests
This commit is contained in:
Loïc Lecrenier
committed by
Loïc Lecrenier
Loïc Lecrenier
parent
9026867d17
commit
b2f01ad204
@ -68,10 +68,244 @@ impl<'i> FacetsUpdate<'i> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// here I want to create a benchmark
|
||||
// to find out at which point it is faster to do it incrementally
|
||||
pub(crate) mod tests {
|
||||
use super::bulk::FacetsUpdateBulkInner;
|
||||
use crate::heed_codec::facet::{
|
||||
ByteSliceRef, FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec,
|
||||
};
|
||||
use crate::search::facet::get_highest_level;
|
||||
use crate::snapshot_tests::display_bitmap;
|
||||
use crate::update::FacetsUpdateIncrementalInner;
|
||||
use crate::CboRoaringBitmapCodec;
|
||||
use heed::types::ByteSlice;
|
||||
use heed::{BytesDecode, BytesEncode, Env, RoTxn, RwTxn};
|
||||
use roaring::RoaringBitmap;
|
||||
use std::fmt::Display;
|
||||
use std::marker::PhantomData;
|
||||
use std::rc::Rc;
|
||||
|
||||
#[test]
|
||||
fn update() {}
|
||||
// A dummy index that only contains the facet database, used for testing
|
||||
pub struct FacetIndex<BoundCodec>
|
||||
where
|
||||
for<'a> BoundCodec:
|
||||
BytesEncode<'a> + BytesDecode<'a, DItem = <BoundCodec as BytesEncode<'a>>::EItem>,
|
||||
{
|
||||
pub env: Env,
|
||||
pub content: heed::Database<FacetGroupKeyCodec<ByteSliceRef>, FacetGroupValueCodec>,
|
||||
pub group_size: u8,
|
||||
pub min_level_size: u8,
|
||||
pub max_group_size: u8,
|
||||
_tempdir: Rc<tempfile::TempDir>,
|
||||
_phantom: PhantomData<BoundCodec>,
|
||||
}
|
||||
|
||||
impl<BoundCodec> FacetIndex<BoundCodec>
|
||||
where
|
||||
for<'a> BoundCodec:
|
||||
BytesEncode<'a> + BytesDecode<'a, DItem = <BoundCodec as BytesEncode<'a>>::EItem>,
|
||||
{
|
||||
#[cfg(all(test, fuzzing))]
|
||||
pub fn open_from_tempdir(
|
||||
tempdir: Rc<tempfile::TempDir>,
|
||||
group_size: u8,
|
||||
max_group_size: u8,
|
||||
min_level_size: u8,
|
||||
) -> FacetIndex<BoundCodec> {
|
||||
let group_size = std::cmp::min(127, std::cmp::max(group_size, 2)); // 2 <= x <= 127
|
||||
let max_group_size = std::cmp::min(127, std::cmp::max(group_size * 2, max_group_size)); // 2*group_size <= x <= 127
|
||||
let min_level_size = std::cmp::max(1, min_level_size); // 1 <= x <= inf
|
||||
|
||||
let mut options = heed::EnvOpenOptions::new();
|
||||
let options = options.map_size(4096 * 4 * 10 * 100);
|
||||
unsafe {
|
||||
options.flag(heed::flags::Flags::MdbAlwaysFreePages);
|
||||
}
|
||||
let env = options.open(tempdir.path()).unwrap();
|
||||
let content = env.open_database(None).unwrap().unwrap();
|
||||
|
||||
FacetIndex {
|
||||
db: Database {
|
||||
content,
|
||||
group_size,
|
||||
max_group_size,
|
||||
min_level_size,
|
||||
_tempdir: tempdir,
|
||||
},
|
||||
env,
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
pub fn new(
|
||||
group_size: u8,
|
||||
max_group_size: u8,
|
||||
min_level_size: u8,
|
||||
) -> FacetIndex<BoundCodec> {
|
||||
let group_size = std::cmp::min(127, std::cmp::max(group_size, 2)); // 2 <= x <= 127
|
||||
let max_group_size = std::cmp::min(127, std::cmp::max(group_size * 2, max_group_size)); // 2*group_size <= x <= 127
|
||||
let min_level_size = std::cmp::max(1, min_level_size); // 1 <= x <= inf
|
||||
let mut options = heed::EnvOpenOptions::new();
|
||||
let options = options.map_size(4096 * 4 * 100);
|
||||
let tempdir = tempfile::TempDir::new().unwrap();
|
||||
let env = options.open(tempdir.path()).unwrap();
|
||||
let content = env.create_database(None).unwrap();
|
||||
|
||||
FacetIndex {
|
||||
content,
|
||||
group_size,
|
||||
max_group_size,
|
||||
min_level_size,
|
||||
_tempdir: Rc::new(tempdir),
|
||||
env,
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
pub fn insert<'a>(
|
||||
&self,
|
||||
wtxn: &'a mut RwTxn,
|
||||
field_id: u16,
|
||||
key: &'a <BoundCodec as BytesEncode<'a>>::EItem,
|
||||
docids: &RoaringBitmap,
|
||||
) {
|
||||
let update = FacetsUpdateIncrementalInner {
|
||||
db: self.content,
|
||||
group_size: self.group_size,
|
||||
min_level_size: self.min_level_size,
|
||||
max_group_size: self.max_group_size,
|
||||
};
|
||||
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
|
||||
update.insert(wtxn, field_id, &key_bytes, docids).unwrap();
|
||||
}
|
||||
pub fn delete<'a>(
|
||||
&self,
|
||||
wtxn: &'a mut RwTxn,
|
||||
field_id: u16,
|
||||
key: &'a <BoundCodec as BytesEncode<'a>>::EItem,
|
||||
value: u32,
|
||||
) {
|
||||
let update = FacetsUpdateIncrementalInner {
|
||||
db: self.content,
|
||||
group_size: self.group_size,
|
||||
min_level_size: self.min_level_size,
|
||||
max_group_size: self.max_group_size,
|
||||
};
|
||||
let key_bytes = BoundCodec::bytes_encode(&key).unwrap();
|
||||
update.delete(wtxn, field_id, &key_bytes, value).unwrap();
|
||||
}
|
||||
|
||||
pub fn bulk_insert<'a, 'b>(
|
||||
&self,
|
||||
wtxn: &'a mut RwTxn,
|
||||
field_ids: &[u16],
|
||||
els: impl IntoIterator<
|
||||
Item = &'a ((u16, <BoundCodec as BytesEncode<'a>>::EItem), RoaringBitmap),
|
||||
>,
|
||||
) where
|
||||
for<'c> <BoundCodec as BytesEncode<'c>>::EItem: Sized,
|
||||
{
|
||||
let mut new_data = vec![];
|
||||
let mut writer = grenad::Writer::new(&mut new_data);
|
||||
for ((field_id, left_bound), docids) in els {
|
||||
let left_bound_bytes = BoundCodec::bytes_encode(left_bound).unwrap().into_owned();
|
||||
let key: FacetGroupKey<&[u8]> =
|
||||
FacetGroupKey { field_id: *field_id, level: 0, left_bound: &left_bound_bytes };
|
||||
let key = FacetGroupKeyCodec::<ByteSliceRef>::bytes_encode(&key).unwrap();
|
||||
let value = CboRoaringBitmapCodec::bytes_encode(&docids).unwrap();
|
||||
writer.insert(&key, &value).unwrap();
|
||||
}
|
||||
writer.finish().unwrap();
|
||||
let reader = grenad::Reader::new(std::io::Cursor::new(new_data)).unwrap();
|
||||
|
||||
let update = FacetsUpdateBulkInner {
|
||||
db: self.content,
|
||||
new_data: Some(reader),
|
||||
group_size: self.group_size,
|
||||
min_level_size: self.min_level_size,
|
||||
};
|
||||
|
||||
update.update(wtxn, field_ids, |_, _, _| Ok(())).unwrap();
|
||||
}
|
||||
|
||||
pub fn verify_structure_validity(&self, txn: &RoTxn, field_id: u16) {
|
||||
let mut field_id_prefix = vec![];
|
||||
field_id_prefix.extend_from_slice(&field_id.to_be_bytes());
|
||||
|
||||
let highest_level = get_highest_level(txn, self.content, field_id).unwrap();
|
||||
|
||||
for level_no in (1..=highest_level).rev() {
|
||||
let mut level_no_prefix = vec![];
|
||||
level_no_prefix.extend_from_slice(&field_id.to_be_bytes());
|
||||
level_no_prefix.push(level_no);
|
||||
|
||||
let mut iter = self
|
||||
.content
|
||||
.as_polymorph()
|
||||
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(txn, &level_no_prefix)
|
||||
.unwrap();
|
||||
while let Some(el) = iter.next() {
|
||||
let (key, value) = el.unwrap();
|
||||
let key = FacetGroupKeyCodec::<ByteSliceRef>::bytes_decode(&key).unwrap();
|
||||
|
||||
let mut prefix_start_below = vec![];
|
||||
prefix_start_below.extend_from_slice(&field_id.to_be_bytes());
|
||||
prefix_start_below.push(level_no - 1);
|
||||
prefix_start_below.extend_from_slice(&key.left_bound);
|
||||
|
||||
let start_below = {
|
||||
let mut start_below_iter = self
|
||||
.content
|
||||
.as_polymorph()
|
||||
.prefix_iter::<_, ByteSlice, FacetGroupValueCodec>(
|
||||
txn,
|
||||
&prefix_start_below,
|
||||
)
|
||||
.unwrap();
|
||||
let (key_bytes, _) = start_below_iter.next().unwrap().unwrap();
|
||||
FacetGroupKeyCodec::<ByteSliceRef>::bytes_decode(&key_bytes).unwrap()
|
||||
};
|
||||
|
||||
assert!(value.size > 0 && value.size < self.max_group_size);
|
||||
|
||||
let mut actual_size = 0;
|
||||
let mut values_below = RoaringBitmap::new();
|
||||
let mut iter_below = self
|
||||
.content
|
||||
.range(txn, &(start_below..))
|
||||
.unwrap()
|
||||
.take(value.size as usize);
|
||||
while let Some(el) = iter_below.next() {
|
||||
let (_, value) = el.unwrap();
|
||||
actual_size += 1;
|
||||
values_below |= value.bitmap;
|
||||
}
|
||||
assert_eq!(actual_size, value.size, "{key:?} start_below: {start_below:?}");
|
||||
|
||||
assert_eq!(value.bitmap, values_below);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<BoundCodec> Display for FacetIndex<BoundCodec>
|
||||
where
|
||||
for<'a> <BoundCodec as BytesEncode<'a>>::EItem: Sized + Display,
|
||||
for<'a> BoundCodec:
|
||||
BytesEncode<'a> + BytesDecode<'a, DItem = <BoundCodec as BytesEncode<'a>>::EItem>,
|
||||
{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let txn = self.env.read_txn().unwrap();
|
||||
let mut iter = self.content.iter(&txn).unwrap();
|
||||
while let Some(el) = iter.next() {
|
||||
let (key, value) = el.unwrap();
|
||||
let FacetGroupKey { field_id, level, left_bound: bound } = key;
|
||||
let bound = BoundCodec::bytes_decode(bound).unwrap();
|
||||
let FacetGroupValue { size, bitmap } = value;
|
||||
writeln!(
|
||||
f,
|
||||
"{field_id:<2} {level:<2} k{bound:<8} {size:<4} {values:?}",
|
||||
values = display_bitmap(&bitmap)
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user