mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-31 10:50:03 +00:00
format the whole project
This commit is contained in:
@ -1,29 +1,28 @@
|
||||
use std::collections::HashMap;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use roaring::RoaringBitmap;
|
||||
|
||||
use crate::{FieldId, TreeLevel, search::{word_derivations, WordDerivationsCache}};
|
||||
use crate::{Index, DocumentId, Result};
|
||||
|
||||
use super::query_tree::{Operation, PrimitiveQueryPart, Query, QueryKind};
|
||||
use self::asc_desc::AscDesc;
|
||||
use self::attribute::Attribute;
|
||||
use self::exactness::Exactness;
|
||||
use self::r#final::Final;
|
||||
use self::initial::Initial;
|
||||
use self::proximity::Proximity;
|
||||
use self::r#final::Final;
|
||||
use self::typo::Typo;
|
||||
use self::words::Words;
|
||||
use super::query_tree::{Operation, PrimitiveQueryPart, Query, QueryKind};
|
||||
use crate::search::{word_derivations, WordDerivationsCache};
|
||||
use crate::{DocumentId, FieldId, Index, Result, TreeLevel};
|
||||
|
||||
mod asc_desc;
|
||||
mod attribute;
|
||||
mod exactness;
|
||||
pub mod r#final;
|
||||
mod initial;
|
||||
mod proximity;
|
||||
mod typo;
|
||||
mod words;
|
||||
pub mod r#final;
|
||||
|
||||
pub trait Criterion {
|
||||
fn next(&mut self, params: &mut CriterionParameters) -> Result<Option<CriterionResult>>;
|
||||
@ -55,7 +54,7 @@ pub struct CriterionParameters<'a> {
|
||||
#[derive(Debug)]
|
||||
enum Candidates {
|
||||
Allowed(RoaringBitmap),
|
||||
Forbidden(RoaringBitmap)
|
||||
Forbidden(RoaringBitmap),
|
||||
}
|
||||
|
||||
impl Default for Candidates {
|
||||
@ -68,17 +67,55 @@ pub trait Context<'c> {
|
||||
fn documents_ids(&self) -> heed::Result<RoaringBitmap>;
|
||||
fn word_docids(&self, word: &str) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_prefix_docids(&self, word: &str) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_prefix_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_prefix_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn words_fst<'t>(&self) -> &'t fst::Set<Cow<[u8]>>;
|
||||
fn in_prefix_cache(&self, word: &str) -> bool;
|
||||
fn docid_words_positions(&self, docid: DocumentId) -> heed::Result<HashMap<String, RoaringBitmap>>;
|
||||
fn word_position_iterator(&self, word: &str, level: TreeLevel, in_prefix_cache: bool, left: Option<u32>, right: Option<u32>) -> heed::Result<Box<dyn Iterator<Item =heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>> + 'c>>;
|
||||
fn word_position_last_level(&self, word: &str, in_prefix_cache: bool) -> heed::Result<Option<TreeLevel>>;
|
||||
fn docid_words_positions(
|
||||
&self,
|
||||
docid: DocumentId,
|
||||
) -> heed::Result<HashMap<String, RoaringBitmap>>;
|
||||
fn word_position_iterator(
|
||||
&self,
|
||||
word: &str,
|
||||
level: TreeLevel,
|
||||
in_prefix_cache: bool,
|
||||
left: Option<u32>,
|
||||
right: Option<u32>,
|
||||
) -> heed::Result<
|
||||
Box<
|
||||
dyn Iterator<Item = heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>> + 'c,
|
||||
>,
|
||||
>;
|
||||
fn word_position_last_level(
|
||||
&self,
|
||||
word: &str,
|
||||
in_prefix_cache: bool,
|
||||
) -> heed::Result<Option<TreeLevel>>;
|
||||
fn synonyms(&self, word: &str) -> heed::Result<Option<Vec<Vec<String>>>>;
|
||||
fn searchable_fields_ids(&self) -> Result<Vec<FieldId>>;
|
||||
fn field_id_word_count_docids(&self, field_id: FieldId, word_count: u8) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_level_position_docids(&self, word: &str, level: TreeLevel, left: u32, right: u32) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn searchable_fields_ids(&self) -> Result<Vec<FieldId>>;
|
||||
fn field_id_word_count_docids(
|
||||
&self,
|
||||
field_id: FieldId,
|
||||
word_count: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>>;
|
||||
fn word_level_position_docids(
|
||||
&self,
|
||||
word: &str,
|
||||
level: TreeLevel,
|
||||
left: u32,
|
||||
right: u32,
|
||||
) -> heed::Result<Option<RoaringBitmap>>;
|
||||
}
|
||||
|
||||
pub struct CriteriaBuilder<'t> {
|
||||
@ -101,12 +138,22 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
self.index.word_prefix_docids.get(self.rtxn, &word)
|
||||
}
|
||||
|
||||
fn word_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (left, right, proximity);
|
||||
self.index.word_pair_proximity_docids.get(self.rtxn, &key)
|
||||
}
|
||||
|
||||
fn word_prefix_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_prefix_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (left, right, proximity);
|
||||
self.index.word_prefix_pair_proximity_docids.get(self.rtxn, &key)
|
||||
}
|
||||
@ -119,7 +166,10 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
self.words_prefixes_fst.contains(word)
|
||||
}
|
||||
|
||||
fn docid_words_positions(&self, docid: DocumentId) -> heed::Result<HashMap<String, RoaringBitmap>> {
|
||||
fn docid_words_positions(
|
||||
&self,
|
||||
docid: DocumentId,
|
||||
) -> heed::Result<HashMap<String, RoaringBitmap>> {
|
||||
let mut words_positions = HashMap::new();
|
||||
for result in self.index.docid_word_positions.prefix_iter(self.rtxn, &(docid, ""))? {
|
||||
let ((_, word), positions) = result?;
|
||||
@ -134,9 +184,12 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
level: TreeLevel,
|
||||
in_prefix_cache: bool,
|
||||
left: Option<u32>,
|
||||
right: Option<u32>
|
||||
) -> heed::Result<Box<dyn Iterator<Item = heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>> + 'c>>
|
||||
{
|
||||
right: Option<u32>,
|
||||
) -> heed::Result<
|
||||
Box<
|
||||
dyn Iterator<Item = heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>> + 'c,
|
||||
>,
|
||||
> {
|
||||
let range = {
|
||||
let left = left.unwrap_or(u32::min_value());
|
||||
let right = right.unwrap_or(u32::max_value());
|
||||
@ -152,7 +205,11 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
Ok(Box::new(db.range(self.rtxn, &range)?))
|
||||
}
|
||||
|
||||
fn word_position_last_level(&self, word: &str, in_prefix_cache: bool) -> heed::Result<Option<TreeLevel>> {
|
||||
fn word_position_last_level(
|
||||
&self,
|
||||
word: &str,
|
||||
in_prefix_cache: bool,
|
||||
) -> heed::Result<Option<TreeLevel>> {
|
||||
let range = {
|
||||
let left = (word, TreeLevel::min_value(), u32::min_value(), u32::min_value());
|
||||
let right = (word, TreeLevel::max_value(), u32::max_value(), u32::max_value());
|
||||
@ -164,7 +221,9 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
};
|
||||
let last_level = db
|
||||
.remap_data_type::<heed::types::DecodeIgnore>()
|
||||
.range(self.rtxn, &range)?.last().transpose()?
|
||||
.range(self.rtxn, &range)?
|
||||
.last()
|
||||
.transpose()?
|
||||
.map(|((_, level, _, _), _)| level);
|
||||
|
||||
Ok(last_level)
|
||||
@ -181,12 +240,22 @@ impl<'c> Context<'c> for CriteriaBuilder<'c> {
|
||||
}
|
||||
}
|
||||
|
||||
fn field_id_word_count_docids(&self, field_id: FieldId, word_count: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn field_id_word_count_docids(
|
||||
&self,
|
||||
field_id: FieldId,
|
||||
word_count: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (field_id, word_count);
|
||||
self.index.field_id_word_count_docids.get(self.rtxn, &key)
|
||||
}
|
||||
|
||||
fn word_level_position_docids(&self, word: &str, level: TreeLevel, left: u32, right: u32) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_level_position_docids(
|
||||
&self,
|
||||
word: &str,
|
||||
level: TreeLevel,
|
||||
left: u32,
|
||||
right: u32,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (word, level, left, right);
|
||||
self.index.word_level_position_docids.get(self.rtxn, &key)
|
||||
}
|
||||
@ -204,13 +273,13 @@ impl<'t> CriteriaBuilder<'t> {
|
||||
query_tree: Option<Operation>,
|
||||
primitive_query: Option<Vec<PrimitiveQueryPart>>,
|
||||
filtered_candidates: Option<RoaringBitmap>,
|
||||
) -> Result<Final<'t>>
|
||||
{
|
||||
) -> Result<Final<'t>> {
|
||||
use crate::criterion::Criterion as Name;
|
||||
|
||||
let primitive_query = primitive_query.unwrap_or_default();
|
||||
|
||||
let mut criterion = Box::new(Initial::new(query_tree, filtered_candidates)) as Box<dyn Criterion>;
|
||||
let mut criterion =
|
||||
Box::new(Initial::new(query_tree, filtered_candidates)) as Box<dyn Criterion>;
|
||||
for name in self.index.criteria(&self.rtxn)? {
|
||||
criterion = match name {
|
||||
Name::Typo => Box::new(Typo::new(self, criterion)),
|
||||
@ -218,8 +287,12 @@ impl<'t> CriteriaBuilder<'t> {
|
||||
Name::Proximity => Box::new(Proximity::new(self, criterion)),
|
||||
Name::Attribute => Box::new(Attribute::new(self, criterion)),
|
||||
Name::Exactness => Box::new(Exactness::new(self, criterion, &primitive_query)?),
|
||||
Name::Asc(field) => Box::new(AscDesc::asc(&self.index, &self.rtxn, criterion, field)?),
|
||||
Name::Desc(field) => Box::new(AscDesc::desc(&self.index, &self.rtxn, criterion, field)?),
|
||||
Name::Asc(field) => {
|
||||
Box::new(AscDesc::asc(&self.index, &self.rtxn, criterion, field)?)
|
||||
}
|
||||
Name::Desc(field) => {
|
||||
Box::new(AscDesc::desc(&self.index, &self.rtxn, criterion, field)?)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -231,21 +304,20 @@ pub fn resolve_query_tree<'t>(
|
||||
ctx: &'t dyn Context,
|
||||
query_tree: &Operation,
|
||||
wdcache: &mut WordDerivationsCache,
|
||||
) -> Result<RoaringBitmap>
|
||||
{
|
||||
) -> Result<RoaringBitmap> {
|
||||
fn resolve_operation<'t>(
|
||||
ctx: &'t dyn Context,
|
||||
query_tree: &Operation,
|
||||
wdcache: &mut WordDerivationsCache,
|
||||
) -> Result<RoaringBitmap>
|
||||
{
|
||||
use Operation::{And, Phrase, Or, Query};
|
||||
) -> Result<RoaringBitmap> {
|
||||
use Operation::{And, Or, Phrase, Query};
|
||||
|
||||
match query_tree {
|
||||
And(ops) => {
|
||||
let mut ops = ops.iter().map(|op| {
|
||||
resolve_operation(ctx, op, wdcache)
|
||||
}).collect::<Result<Vec<_>>>()?;
|
||||
let mut ops = ops
|
||||
.iter()
|
||||
.map(|op| resolve_operation(ctx, op, wdcache))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
ops.sort_unstable_by_key(|cds| cds.len());
|
||||
|
||||
@ -260,7 +332,7 @@ pub fn resolve_query_tree<'t>(
|
||||
}
|
||||
}
|
||||
Ok(candidates)
|
||||
},
|
||||
}
|
||||
Phrase(words) => {
|
||||
let mut candidates = RoaringBitmap::new();
|
||||
let mut first_loop = true;
|
||||
@ -276,12 +348,12 @@ pub fn resolve_query_tree<'t>(
|
||||
} else {
|
||||
candidates &= pair_docids;
|
||||
}
|
||||
},
|
||||
None => return Ok(RoaringBitmap::new())
|
||||
}
|
||||
None => return Ok(RoaringBitmap::new()),
|
||||
}
|
||||
}
|
||||
Ok(candidates)
|
||||
},
|
||||
}
|
||||
Or(_, ops) => {
|
||||
let mut candidates = RoaringBitmap::new();
|
||||
for op in ops {
|
||||
@ -289,7 +361,7 @@ pub fn resolve_query_tree<'t>(
|
||||
candidates.union_with(&docids);
|
||||
}
|
||||
Ok(candidates)
|
||||
},
|
||||
}
|
||||
Query(q) => Ok(query_docids(ctx, q, wdcache)?),
|
||||
}
|
||||
}
|
||||
@ -297,18 +369,18 @@ pub fn resolve_query_tree<'t>(
|
||||
resolve_operation(ctx, query_tree, wdcache)
|
||||
}
|
||||
|
||||
|
||||
fn all_word_pair_proximity_docids<T: AsRef<str>, U: AsRef<str>>(
|
||||
ctx: &dyn Context,
|
||||
left_words: &[(T, u8)],
|
||||
right_words: &[(U, u8)],
|
||||
proximity: u8
|
||||
) -> Result<RoaringBitmap>
|
||||
{
|
||||
proximity: u8,
|
||||
) -> Result<RoaringBitmap> {
|
||||
let mut docids = RoaringBitmap::new();
|
||||
for (left, _l_typo) in left_words {
|
||||
for (right, _r_typo) in right_words {
|
||||
let current_docids = ctx.word_pair_proximity_docids(left.as_ref(), right.as_ref(), proximity)?.unwrap_or_default();
|
||||
let current_docids = ctx
|
||||
.word_pair_proximity_docids(left.as_ref(), right.as_ref(), proximity)?
|
||||
.unwrap_or_default();
|
||||
docids.union_with(¤t_docids);
|
||||
}
|
||||
}
|
||||
@ -319,8 +391,7 @@ fn query_docids(
|
||||
ctx: &dyn Context,
|
||||
query: &Query,
|
||||
wdcache: &mut WordDerivationsCache,
|
||||
) -> Result<RoaringBitmap>
|
||||
{
|
||||
) -> Result<RoaringBitmap> {
|
||||
match &query.kind {
|
||||
QueryKind::Exact { word, .. } => {
|
||||
if query.prefix && ctx.in_prefix_cache(&word) {
|
||||
@ -336,7 +407,7 @@ fn query_docids(
|
||||
} else {
|
||||
Ok(ctx.word_docids(&word)?.unwrap_or_default())
|
||||
}
|
||||
},
|
||||
}
|
||||
QueryKind::Tolerant { typo, word } => {
|
||||
let words = word_derivations(&word, query.prefix, *typo, ctx.words_fst(), wdcache)?;
|
||||
let mut docids = RoaringBitmap::new();
|
||||
@ -345,7 +416,7 @@ fn query_docids(
|
||||
docids.union_with(¤t_docids);
|
||||
}
|
||||
Ok(docids)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -355,8 +426,7 @@ fn query_pair_proximity_docids(
|
||||
right: &Query,
|
||||
proximity: u8,
|
||||
wdcache: &mut WordDerivationsCache,
|
||||
) -> Result<RoaringBitmap>
|
||||
{
|
||||
) -> Result<RoaringBitmap> {
|
||||
if proximity >= 8 {
|
||||
let mut candidates = query_docids(ctx, left, wdcache)?;
|
||||
let right_candidates = query_docids(ctx, right, wdcache)?;
|
||||
@ -368,20 +438,31 @@ fn query_pair_proximity_docids(
|
||||
match (&left.kind, &right.kind) {
|
||||
(QueryKind::Exact { word: left, .. }, QueryKind::Exact { word: right, .. }) => {
|
||||
if prefix && ctx.in_prefix_cache(&right) {
|
||||
Ok(ctx.word_prefix_pair_proximity_docids(left.as_str(), right.as_str(), proximity)?.unwrap_or_default())
|
||||
Ok(ctx
|
||||
.word_prefix_pair_proximity_docids(left.as_str(), right.as_str(), proximity)?
|
||||
.unwrap_or_default())
|
||||
} else if prefix {
|
||||
let r_words = word_derivations(&right, true, 0, ctx.words_fst(), wdcache)?;
|
||||
all_word_pair_proximity_docids(ctx, &[(left, 0)], &r_words, proximity)
|
||||
} else {
|
||||
Ok(ctx.word_pair_proximity_docids(left.as_str(), right.as_str(), proximity)?.unwrap_or_default())
|
||||
Ok(ctx
|
||||
.word_pair_proximity_docids(left.as_str(), right.as_str(), proximity)?
|
||||
.unwrap_or_default())
|
||||
}
|
||||
},
|
||||
}
|
||||
(QueryKind::Tolerant { typo, word: left }, QueryKind::Exact { word: right, .. }) => {
|
||||
let l_words = word_derivations(&left, false, *typo, ctx.words_fst(), wdcache)?.to_owned();
|
||||
let l_words =
|
||||
word_derivations(&left, false, *typo, ctx.words_fst(), wdcache)?.to_owned();
|
||||
if prefix && ctx.in_prefix_cache(&right) {
|
||||
let mut docids = RoaringBitmap::new();
|
||||
for (left, _) in l_words {
|
||||
let current_docids = ctx.word_prefix_pair_proximity_docids(left.as_ref(), right.as_ref(), proximity)?.unwrap_or_default();
|
||||
let current_docids = ctx
|
||||
.word_prefix_pair_proximity_docids(
|
||||
left.as_ref(),
|
||||
right.as_ref(),
|
||||
proximity,
|
||||
)?
|
||||
.unwrap_or_default();
|
||||
docids.union_with(¤t_docids);
|
||||
}
|
||||
Ok(docids)
|
||||
@ -391,28 +472,36 @@ fn query_pair_proximity_docids(
|
||||
} else {
|
||||
all_word_pair_proximity_docids(ctx, &l_words, &[(right, 0)], proximity)
|
||||
}
|
||||
},
|
||||
}
|
||||
(QueryKind::Exact { word: left, .. }, QueryKind::Tolerant { typo, word: right }) => {
|
||||
let r_words = word_derivations(&right, prefix, *typo, ctx.words_fst(), wdcache)?;
|
||||
all_word_pair_proximity_docids(ctx, &[(left, 0)], &r_words, proximity)
|
||||
},
|
||||
(QueryKind::Tolerant { typo: l_typo, word: left }, QueryKind::Tolerant { typo: r_typo, word: right }) => {
|
||||
let l_words = word_derivations(&left, false, *l_typo, ctx.words_fst(), wdcache)?.to_owned();
|
||||
}
|
||||
(
|
||||
QueryKind::Tolerant { typo: l_typo, word: left },
|
||||
QueryKind::Tolerant { typo: r_typo, word: right },
|
||||
) => {
|
||||
let l_words =
|
||||
word_derivations(&left, false, *l_typo, ctx.words_fst(), wdcache)?.to_owned();
|
||||
let r_words = word_derivations(&right, prefix, *r_typo, ctx.words_fst(), wdcache)?;
|
||||
all_word_pair_proximity_docids(ctx, &l_words, &r_words, proximity)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod test {
|
||||
use maplit::hashmap;
|
||||
use rand::{Rng, SeedableRng, rngs::StdRng};
|
||||
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn s(s: &str) -> String { s.to_string() }
|
||||
use maplit::hashmap;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng, SeedableRng};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn s(s: &str) -> String {
|
||||
s.to_string()
|
||||
}
|
||||
pub struct TestContext<'t> {
|
||||
words_fst: fst::Set<Cow<'t, [u8]>>,
|
||||
word_docids: HashMap<String, RoaringBitmap>,
|
||||
@ -435,12 +524,22 @@ pub mod test {
|
||||
Ok(self.word_prefix_docids.get(&word.to_string()).cloned())
|
||||
}
|
||||
|
||||
fn word_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (left.to_string(), right.to_string(), proximity.into());
|
||||
Ok(self.word_pair_proximity_docids.get(&key).cloned())
|
||||
}
|
||||
|
||||
fn word_prefix_pair_proximity_docids(&self, left: &str, right: &str, proximity: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_prefix_pair_proximity_docids(
|
||||
&self,
|
||||
left: &str,
|
||||
right: &str,
|
||||
proximity: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
let key = (left.to_string(), right.to_string(), proximity.into());
|
||||
Ok(self.word_prefix_pair_proximity_docids.get(&key).cloned())
|
||||
}
|
||||
@ -453,24 +552,44 @@ pub mod test {
|
||||
self.word_prefix_docids.contains_key(&word.to_string())
|
||||
}
|
||||
|
||||
fn docid_words_positions(&self, docid: DocumentId) -> heed::Result<HashMap<String, RoaringBitmap>> {
|
||||
fn docid_words_positions(
|
||||
&self,
|
||||
docid: DocumentId,
|
||||
) -> heed::Result<HashMap<String, RoaringBitmap>> {
|
||||
if let Some(docid_words) = self.docid_words.get(&docid) {
|
||||
Ok(docid_words
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i,w)| (w.clone(), RoaringBitmap::from_sorted_iter(std::iter::once(i as u32))))
|
||||
.collect()
|
||||
)
|
||||
.map(|(i, w)| {
|
||||
(w.clone(), RoaringBitmap::from_sorted_iter(std::iter::once(i as u32)))
|
||||
})
|
||||
.collect())
|
||||
} else {
|
||||
Ok(HashMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
fn word_position_iterator(&self, _word: &str, _level: TreeLevel, _in_prefix_cache: bool, _left: Option<u32>, _right: Option<u32>) -> heed::Result<Box<dyn Iterator<Item =heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>> + 'c>> {
|
||||
fn word_position_iterator(
|
||||
&self,
|
||||
_word: &str,
|
||||
_level: TreeLevel,
|
||||
_in_prefix_cache: bool,
|
||||
_left: Option<u32>,
|
||||
_right: Option<u32>,
|
||||
) -> heed::Result<
|
||||
Box<
|
||||
dyn Iterator<Item = heed::Result<((&'c str, TreeLevel, u32, u32), RoaringBitmap)>>
|
||||
+ 'c,
|
||||
>,
|
||||
> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn word_position_last_level(&self, _word: &str, _in_prefix_cache: bool) -> heed::Result<Option<TreeLevel>> {
|
||||
fn word_position_last_level(
|
||||
&self,
|
||||
_word: &str,
|
||||
_in_prefix_cache: bool,
|
||||
) -> heed::Result<Option<TreeLevel>> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
@ -478,15 +597,25 @@ pub mod test {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn searchable_fields_ids(&self) -> Result<Vec<FieldId>> {
|
||||
fn searchable_fields_ids(&self) -> Result<Vec<FieldId>> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn word_level_position_docids(&self, _word: &str, _level: TreeLevel, _left: u32, _right: u32) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn word_level_position_docids(
|
||||
&self,
|
||||
_word: &str,
|
||||
_level: TreeLevel,
|
||||
_left: u32,
|
||||
_right: u32,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn field_id_word_count_docids(&self, _field_id: FieldId, _word_count: u8) -> heed::Result<Option<RoaringBitmap>> {
|
||||
fn field_id_word_count_docids(
|
||||
&self,
|
||||
_field_id: FieldId,
|
||||
_word_count: u8,
|
||||
) -> heed::Result<Option<RoaringBitmap>> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
@ -506,7 +635,7 @@ pub mod test {
|
||||
RoaringBitmap::from_sorted_iter(values.into_iter())
|
||||
}
|
||||
|
||||
let word_docids = hashmap!{
|
||||
let word_docids = hashmap! {
|
||||
s("hello") => random_postings(rng, 1500),
|
||||
s("hi") => random_postings(rng, 4000),
|
||||
s("word") => random_postings(rng, 2500),
|
||||
@ -530,7 +659,7 @@ pub mod test {
|
||||
}
|
||||
}
|
||||
|
||||
let word_prefix_docids = hashmap!{
|
||||
let word_prefix_docids = hashmap! {
|
||||
s("h") => &word_docids[&s("hello")] | &word_docids[&s("hi")],
|
||||
s("wor") => &word_docids[&s("word")] | &word_docids[&s("world")],
|
||||
s("20") => &word_docids[&s("2020")] | &word_docids[&s("2021")],
|
||||
@ -540,7 +669,9 @@ pub mod test {
|
||||
let mut word_prefix_pair_proximity_docids = HashMap::new();
|
||||
for (lword, lcandidates) in &word_docids {
|
||||
for (rword, rcandidates) in &word_docids {
|
||||
if lword == rword { continue }
|
||||
if lword == rword {
|
||||
continue;
|
||||
}
|
||||
let candidates = lcandidates & rcandidates;
|
||||
for candidate in candidates {
|
||||
if let Some(docid_words) = docid_words.get(&candidate) {
|
||||
@ -551,24 +682,31 @@ pub mod test {
|
||||
} else {
|
||||
(s(lword), s(rword), (lposition - rposition + 1) as i32)
|
||||
};
|
||||
let docids = word_pair_proximity_docids.entry(key).or_insert(RoaringBitmap::new());
|
||||
let docids = word_pair_proximity_docids
|
||||
.entry(key)
|
||||
.or_insert(RoaringBitmap::new());
|
||||
docids.push(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (pword, pcandidates) in &word_prefix_docids {
|
||||
if lword.starts_with(pword) { continue }
|
||||
if lword.starts_with(pword) {
|
||||
continue;
|
||||
}
|
||||
let candidates = lcandidates & pcandidates;
|
||||
for candidate in candidates {
|
||||
if let Some(docid_words) = docid_words.get(&candidate) {
|
||||
let lposition = docid_words.iter().position(|w| w == lword).unwrap();
|
||||
let rposition = docid_words.iter().position(|w| w.starts_with(pword)).unwrap();
|
||||
let rposition =
|
||||
docid_words.iter().position(|w| w.starts_with(pword)).unwrap();
|
||||
let key = if lposition < rposition {
|
||||
(s(lword), s(pword), (rposition - lposition) as i32)
|
||||
} else {
|
||||
(s(lword), s(pword), (lposition - rposition + 1) as i32)
|
||||
};
|
||||
let docids = word_prefix_pair_proximity_docids.entry(key).or_insert(RoaringBitmap::new());
|
||||
let docids = word_prefix_pair_proximity_docids
|
||||
.entry(key)
|
||||
.or_insert(RoaringBitmap::new());
|
||||
docids.push(candidate);
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user