snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-31 02:40:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Intern more values

Browse Source
This commit is contained in:
Loïc Lecrenier
2023-03-09 11:12:31 +01:00
parent 2099991dd1
commit 0465ba4a05
12 changed files with 879 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
milli/src/search/new/ranking_rule_graph/proximity/build.rs
View File

@ -14,29 +14,33 @@ pub fn visit_from_node(
from_node: &QueryNode,
) -> Result<Option<(WordDerivations, i8)>> {
Ok(Some(match from_node {
QueryNode::Term(LocatedQueryTerm { value: value1, positions: pos1 }) => match value1 {
QueryTerm::Word { derivations } => (derivations.clone(), *pos1.end()),
QueryTerm::Phrase { phrase: phrase1 } => {
let phrase1 = ctx.phrase_interner.get(*phrase1);
if let Some(original) = *phrase1.words.last().unwrap() {
(
WordDerivations {
original,
zero_typo: Box::new([original]),
one_typo: Box::new([]),
two_typos: Box::new([]),
use_prefix_db: false,
synonyms: Box::new([]),
split_words: None,
},
*pos1.end(),
)
} else {
// No word pairs if the phrase does not have a regular word as its last term
return Ok(None);
QueryNode::Term(LocatedQueryTerm { value: value1, positions: pos1 }) => {
match value1 {
QueryTerm::Word { derivations } => {
(ctx.derivations_interner.get(*derivations).clone(), *pos1.end())
}
QueryTerm::Phrase { phrase: phrase1 } => {
let phrase1 = ctx.phrase_interner.get(*phrase1);
if let Some(original) = *phrase1.words.last().unwrap() {
(
WordDerivations {
original,
zero_typo: Box::new([original]),
one_typo: Box::new([]),
two_typos: Box::new([]),
use_prefix_db: false,
synonyms: Box::new([]),
split_words: None,
},
*pos1.end(),
)
} else {
// No word pairs if the phrase does not have a regular word as its last term
return Ok(None);
}
}
}
},
}
QueryNode::Start => (
WordDerivations {
original: ctx.word_interner.insert(String::new()),
@ -58,6 +62,10 @@ pub fn visit_to_node<'search, 'from_data>(
to_node: &QueryNode,
from_node_data: &'from_data (WordDerivations, i8),
) -> Result<Vec<(u8, EdgeCondition<ProximityEdge>)>> {
let SearchContext { index, txn, db_cache, word_interner, derivations_interner, .. } = ctx;
// IMPORTANT! TODO: split words support
let (derivations1, pos1) = from_node_data;
let term2 = match &to_node {
QueryNode::End => return Ok(vec![(0, EdgeCondition::Unconditional)]),
@ -67,7 +75,9 @@ pub fn visit_to_node<'search, 'from_data>(
let LocatedQueryTerm { value: value2, positions: pos2 } = term2;
let (derivations2, pos2, ngram_len2) = match value2 {
QueryTerm::Word { derivations } => (derivations.clone(), *pos2.start(), pos2.len()),
QueryTerm::Word { derivations } => {
(derivations_interner.get(*derivations).clone(), *pos2.start(), pos2.len())
}
QueryTerm::Phrase { phrase: phrase2 } => {
let phrase2 = ctx.phrase_interner.get(*phrase2);
if let Some(original) = *phrase2.words.first().unwrap() {
@ -105,7 +115,8 @@ pub fn visit_to_node<'search, 'from_data>(
// left term cannot be a prefix
assert!(!updb1);
let derivations1 = derivations1.all_derivations_except_prefix_db();
// TODO: IMPORTANT! split words and synonyms support
let derivations1 = derivations1.all_single_word_derivations_except_prefix_db();
// TODO: eventually, we want to get rid of the uses from `orginal`
let mut cost_proximity_word_pairs = BTreeMap::<u8, BTreeMap<u8, Vec<WordPair>>>::new();
@ -115,8 +126,11 @@ pub fn visit_to_node<'search, 'from_data>(
let cost = (proximity + ngram_len2 - 1) as u8;
// TODO: if we had access to the universe here, we could already check whether
// the bitmap corresponding to this word pair is disjoint with the universe or not
if ctx
if db_cache
.get_word_prefix_pair_proximity_docids(
index,
txn,
word_interner,
word1,
derivations2.original,
proximity as u8,
@ -133,8 +147,11 @@ pub fn visit_to_node<'search, 'from_data>(
right_prefix: derivations2.original,
});
}
if ctx
if db_cache
.get_prefix_word_pair_proximity_docids(
index,
txn,
word_interner,
derivations2.original,
word1,
proximity as u8 - 1,
@ -155,14 +172,30 @@ pub fn visit_to_node<'search, 'from_data>(
}
}
let derivations2 = derivations2.all_derivations_except_prefix_db();
// TODO: add safeguard in case the cartesian product is too large?
// TODO: important! support split words and synonyms as well
let derivations2 = derivations2.all_single_word_derivations_except_prefix_db();
// TODO: add safeguard in case the cartesian product is too large!
// even if we restrict the word derivations to a maximum of 100, the size of the
// caterisan product could reach a maximum of 10_000 derivations, which is way too much.
// mMaybe prioritise the product of zero typo derivations, then the product of zero-typo/one-typo
// + one-typo/zero-typo, then one-typo/one-typo, then ... until an arbitrary limit has been
// reached
let product_derivations = derivations1.cartesian_product(derivations2);
for (word1, word2) in product_derivations {
for proximity in 1..=(8 - ngram_len2) {
let cost = (proximity + ngram_len2 - 1) as u8;
if ctx.get_word_pair_proximity_docids(word1, word2, proximity as u8)?.is_some() {
if db_cache
.get_word_pair_proximity_docids(
index,
txn,
word_interner,
word1,
word2,
proximity as u8,
)?
.is_some()
{
cost_proximity_word_pairs
.entry(cost)
.or_default()
@ -171,7 +204,16 @@ pub fn visit_to_node<'search, 'from_data>(
.push(WordPair::Words { left: word1, right: word2 });
}
if proximity > 1
&& ctx.get_word_pair_proximity_docids(word2, word1, proximity as u8 - 1)?.is_some()
&& db_cache
.get_word_pair_proximity_docids(
index,
txn,
word_interner,
word2,
word1,
proximity as u8 - 1,
)?
.is_some()
{
cost_proximity_word_pairs
.entry(cost)

36
milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs
View File

@ -9,19 +9,37 @@ pub fn compute_docids<'search>(
edge: &ProximityEdge,
universe: &RoaringBitmap,
) -> Result<RoaringBitmap> {
let SearchContext { index, txn, db_cache, word_interner, .. } = ctx;
let ProximityEdge { pairs, proximity } = edge;
let mut pair_docids = RoaringBitmap::new();
for pair in pairs.iter() {
let bytes = match pair {
WordPair::Words { left, right } => {
ctx.get_word_pair_proximity_docids(*left, *right, *proximity)
}
WordPair::WordPrefix { left, right_prefix } => {
ctx.get_word_prefix_pair_proximity_docids(*left, *right_prefix, *proximity)
}
WordPair::WordPrefixSwapped { left_prefix, right } => {
ctx.get_prefix_word_pair_proximity_docids(*left_prefix, *right, *proximity)
}
WordPair::Words { left, right } => db_cache.get_word_pair_proximity_docids(
index,
txn,
word_interner,
*left,
*right,
*proximity,
),
WordPair::WordPrefix { left, right_prefix } => db_cache
.get_word_prefix_pair_proximity_docids(
index,
txn,
word_interner,
*left,
*right_prefix,
*proximity,
),
WordPair::WordPrefixSwapped { left_prefix, right } => db_cache
.get_prefix_word_pair_proximity_docids(
index,
txn,
word_interner,
*left_prefix,
*right,
*proximity,
),
}?;
// TODO: deserialize bitmap within a universe, and (maybe) using a bump allocator?
let bitmap = universe

147
milli/src/search/new/ranking_rule_graph/typo/mod.rs
View File

@ -1,4 +1,3 @@
use heed::BytesDecode;
use roaring::RoaringBitmap;
use super::empty_paths_cache::EmptyPathsCache;
@ -6,15 +5,14 @@ use super::{EdgeCondition, RankingRuleGraph, RankingRuleGraphTrait};
use crate::search::new::interner::Interned;
use crate::search::new::logger::SearchLogger;
use crate::search::new::query_term::{LocatedQueryTerm, Phrase, QueryTerm, WordDerivations};
use crate::search::new::resolve_query_graph::resolve_phrase;
use crate::search::new::small_bitmap::SmallBitmap;
use crate::search::new::{QueryGraph, QueryNode, SearchContext};
use crate::{Result, RoaringBitmapCodec};
use crate::Result;
#[derive(Clone)]
pub enum TypoEdge {
Phrase { phrase: Interned<Phrase> },
Word { derivations: WordDerivations, nbr_typos: u8 },
Word { derivations: Interned<WordDerivations>, nbr_typos: u8 },
}
pub enum TypoGraph {}
@ -35,32 +33,37 @@ impl RankingRuleGraphTrait for TypoGraph {
edge: &Self::EdgeCondition,
universe: &RoaringBitmap,
) -> Result<RoaringBitmap> {
let SearchContext {
index,
txn,
db_cache,
word_interner,
phrase_interner,
derivations_interner,
query_term_docids,
} = ctx;
match edge {
TypoEdge::Phrase { phrase } => resolve_phrase(ctx, *phrase),
TypoEdge::Word { derivations, nbr_typos } => {
let words = match nbr_typos {
0 => &derivations.zero_typo,
1 => &derivations.one_typo,
2 => &derivations.two_typos,
_ => panic!(),
};
let mut docids = RoaringBitmap::new();
for word in words.iter().copied() {
let Some(bytes) = ctx.get_word_docids(word)? else { continue };
// TODO: deserialize bitmap within a universe
let bitmap = universe
& RoaringBitmapCodec::bytes_decode(bytes).ok_or(heed::Error::Decoding)?;
docids |= bitmap;
}
if *nbr_typos == 0 {
if let Some(bytes) = ctx.get_word_prefix_docids(derivations.original)? {
// TODO: deserialize bitmap within a universe
let bitmap = universe
& RoaringBitmapCodec::bytes_decode(bytes)
.ok_or(heed::Error::Decoding)?;
docids |= bitmap;
}
}
&TypoEdge::Phrase { phrase } => Ok(universe
& query_term_docids.get_phrase_docids(
index,
txn,
db_cache,
word_interner,
phrase_interner,
phrase,
)?),
TypoEdge::Word { derivations, .. } => {
let docids = universe
& query_term_docids.get_word_derivations_docids(
index,
txn,
db_cache,
word_interner,
derivations_interner,
phrase_interner,
*derivations,
)?;
Ok(docids)
}
}
@ -74,43 +77,71 @@ impl RankingRuleGraphTrait for TypoGraph {
}
fn build_step_visit_destination_node<'from_data, 'search: 'from_data>(
_ctx: &mut SearchContext<'search>,
ctx: &mut SearchContext<'search>,
to_node: &QueryNode,
_from_node_data: &'from_data Self::BuildVisitedFromNode,
) -> Result<Vec<(u8, EdgeCondition<Self::EdgeCondition>)>> {
let SearchContext { derivations_interner, .. } = ctx;
match to_node {
QueryNode::Term(LocatedQueryTerm { value, .. }) => match value {
&QueryTerm::Phrase { phrase } => {
QueryNode::Term(LocatedQueryTerm { value, .. }) => match *value {
QueryTerm::Phrase { phrase } => {
Ok(vec![(0, EdgeCondition::Conditional(TypoEdge::Phrase { phrase }))])
}
QueryTerm::Word { derivations } => {
let mut edges = vec![];
if !derivations.zero_typo.is_empty() || derivations.use_prefix_db {
edges.push((
0,
EdgeCondition::Conditional(TypoEdge::Word {
derivations: derivations.clone(),
nbr_typos: 0,
}),
))
}
if !derivations.one_typo.is_empty() {
edges.push((
1,
EdgeCondition::Conditional(TypoEdge::Word {
derivations: derivations.clone(),
nbr_typos: 1,
}),
))
}
if !derivations.two_typos.is_empty() {
edges.push((
2,
EdgeCondition::Conditional(TypoEdge::Word {
derivations: derivations.clone(),
nbr_typos: 2,
}),
))
for nbr_typos in 0..=2 {
let derivations = derivations_interner.get(derivations).clone();
let new_derivations = match nbr_typos {
0 => {
// TODO: think about how split words and synonyms should be handled here
// TODO: what about ngrams?
// Maybe 2grams should have one typo by default and 3grams 2 typos by default
WordDerivations {
original: derivations.original,
synonyms: derivations.synonyms,
split_words: None,
zero_typo: derivations.zero_typo,
one_typo: Box::new([]),
two_typos: Box::new([]),
use_prefix_db: derivations.use_prefix_db,
}
}
1 => {
// What about split words and synonyms here?
WordDerivations {
original: derivations.original,
synonyms: Box::new([]),
split_words: derivations.split_words,
zero_typo: Box::new([]),
one_typo: derivations.one_typo,
two_typos: Box::new([]),
use_prefix_db: false, // false because all items from use_prefix_db haev 0 typos
}
}
2 => {
// What about split words and synonyms here?
WordDerivations {
original: derivations.original,
synonyms: Box::new([]),
split_words: None,
zero_typo: Box::new([]),
one_typo: Box::new([]),
two_typos: derivations.two_typos,
use_prefix_db: false, // false because all items from use_prefix_db haev 0 typos
}
}
_ => panic!(),
};
if !new_derivations.is_empty() {
edges.push((
nbr_typos,
EdgeCondition::Conditional(TypoEdge::Word {
derivations: derivations_interner.insert(new_derivations),
nbr_typos,
}),
))
}
}
Ok(edges)
}