snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-31 10:50:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Intern more values

Browse Source
This commit is contained in:
Loïc Lecrenier
2023-03-09 11:12:31 +01:00
parent 2099991dd1
commit 0465ba4a05
12 changed files with 879 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
milli/src/search/new/ranking_rule_graph/proximity/build.rs
View File

@ -14,29 +14,33 @@ pub fn visit_from_node(
from_node: &QueryNode,
) -> Result<Option<(WordDerivations, i8)>> {
Ok(Some(match from_node {
QueryNode::Term(LocatedQueryTerm { value: value1, positions: pos1 }) => match value1 {
QueryTerm::Word { derivations } => (derivations.clone(), *pos1.end()),
QueryTerm::Phrase { phrase: phrase1 } => {
let phrase1 = ctx.phrase_interner.get(*phrase1);
if let Some(original) = *phrase1.words.last().unwrap() {
(
WordDerivations {
original,
zero_typo: Box::new([original]),
one_typo: Box::new([]),
two_typos: Box::new([]),
use_prefix_db: false,
synonyms: Box::new([]),
split_words: None,
},
*pos1.end(),
)
} else {
// No word pairs if the phrase does not have a regular word as its last term
return Ok(None);
QueryNode::Term(LocatedQueryTerm { value: value1, positions: pos1 }) => {
match value1 {
QueryTerm::Word { derivations } => {
(ctx.derivations_interner.get(*derivations).clone(), *pos1.end())
}
QueryTerm::Phrase { phrase: phrase1 } => {
let phrase1 = ctx.phrase_interner.get(*phrase1);
if let Some(original) = *phrase1.words.last().unwrap() {
(
WordDerivations {
original,
zero_typo: Box::new([original]),
one_typo: Box::new([]),
two_typos: Box::new([]),
use_prefix_db: false,
synonyms: Box::new([]),
split_words: None,
},
*pos1.end(),
)
} else {
// No word pairs if the phrase does not have a regular word as its last term
return Ok(None);
}
}
}
},
}
QueryNode::Start => (
WordDerivations {
original: ctx.word_interner.insert(String::new()),
@ -58,6 +62,10 @@ pub fn visit_to_node<'search, 'from_data>(
to_node: &QueryNode,
from_node_data: &'from_data (WordDerivations, i8),
) -> Result<Vec<(u8, EdgeCondition<ProximityEdge>)>> {
let SearchContext { index, txn, db_cache, word_interner, derivations_interner, .. } = ctx;
// IMPORTANT! TODO: split words support
let (derivations1, pos1) = from_node_data;
let term2 = match &to_node {
QueryNode::End => return Ok(vec![(0, EdgeCondition::Unconditional)]),
@ -67,7 +75,9 @@ pub fn visit_to_node<'search, 'from_data>(
let LocatedQueryTerm { value: value2, positions: pos2 } = term2;
let (derivations2, pos2, ngram_len2) = match value2 {
QueryTerm::Word { derivations } => (derivations.clone(), *pos2.start(), pos2.len()),
QueryTerm::Word { derivations } => {
(derivations_interner.get(*derivations).clone(), *pos2.start(), pos2.len())
}
QueryTerm::Phrase { phrase: phrase2 } => {
let phrase2 = ctx.phrase_interner.get(*phrase2);
if let Some(original) = *phrase2.words.first().unwrap() {
@ -105,7 +115,8 @@ pub fn visit_to_node<'search, 'from_data>(
// left term cannot be a prefix
assert!(!updb1);
let derivations1 = derivations1.all_derivations_except_prefix_db();
// TODO: IMPORTANT! split words and synonyms support
let derivations1 = derivations1.all_single_word_derivations_except_prefix_db();
// TODO: eventually, we want to get rid of the uses from `orginal`
let mut cost_proximity_word_pairs = BTreeMap::<u8, BTreeMap<u8, Vec<WordPair>>>::new();
@ -115,8 +126,11 @@ pub fn visit_to_node<'search, 'from_data>(
let cost = (proximity + ngram_len2 - 1) as u8;
// TODO: if we had access to the universe here, we could already check whether
// the bitmap corresponding to this word pair is disjoint with the universe or not
if ctx
if db_cache
.get_word_prefix_pair_proximity_docids(
index,
txn,
word_interner,
word1,
derivations2.original,
proximity as u8,
@ -133,8 +147,11 @@ pub fn visit_to_node<'search, 'from_data>(
right_prefix: derivations2.original,
});
}
if ctx
if db_cache
.get_prefix_word_pair_proximity_docids(
index,
txn,
word_interner,
derivations2.original,
word1,
proximity as u8 - 1,
@ -155,14 +172,30 @@ pub fn visit_to_node<'search, 'from_data>(
}
}
let derivations2 = derivations2.all_derivations_except_prefix_db();
// TODO: add safeguard in case the cartesian product is too large?
// TODO: important! support split words and synonyms as well
let derivations2 = derivations2.all_single_word_derivations_except_prefix_db();
// TODO: add safeguard in case the cartesian product is too large!
// even if we restrict the word derivations to a maximum of 100, the size of the
// caterisan product could reach a maximum of 10_000 derivations, which is way too much.
// mMaybe prioritise the product of zero typo derivations, then the product of zero-typo/one-typo
// + one-typo/zero-typo, then one-typo/one-typo, then ... until an arbitrary limit has been
// reached
let product_derivations = derivations1.cartesian_product(derivations2);
for (word1, word2) in product_derivations {
for proximity in 1..=(8 - ngram_len2) {
let cost = (proximity + ngram_len2 - 1) as u8;
if ctx.get_word_pair_proximity_docids(word1, word2, proximity as u8)?.is_some() {
if db_cache
.get_word_pair_proximity_docids(
index,
txn,
word_interner,
word1,
word2,
proximity as u8,
)?
.is_some()
{
cost_proximity_word_pairs
.entry(cost)
.or_default()
@ -171,7 +204,16 @@ pub fn visit_to_node<'search, 'from_data>(
.push(WordPair::Words { left: word1, right: word2 });
}
if proximity > 1
&& ctx.get_word_pair_proximity_docids(word2, word1, proximity as u8 - 1)?.is_some()
&& db_cache
.get_word_pair_proximity_docids(
index,
txn,
word_interner,
word2,
word1,
proximity as u8 - 1,
)?
.is_some()
{
cost_proximity_word_pairs
.entry(cost)

36
milli/src/search/new/ranking_rule_graph/proximity/compute_docids.rs
View File

@ -9,19 +9,37 @@ pub fn compute_docids<'search>(
edge: &ProximityEdge,
universe: &RoaringBitmap,
) -> Result<RoaringBitmap> {
let SearchContext { index, txn, db_cache, word_interner, .. } = ctx;
let ProximityEdge { pairs, proximity } = edge;
let mut pair_docids = RoaringBitmap::new();
for pair in pairs.iter() {
let bytes = match pair {
WordPair::Words { left, right } => {
ctx.get_word_pair_proximity_docids(*left, *right, *proximity)
}
WordPair::WordPrefix { left, right_prefix } => {
ctx.get_word_prefix_pair_proximity_docids(*left, *right_prefix, *proximity)
}
WordPair::WordPrefixSwapped { left_prefix, right } => {
ctx.get_prefix_word_pair_proximity_docids(*left_prefix, *right, *proximity)
}
WordPair::Words { left, right } => db_cache.get_word_pair_proximity_docids(
index,
txn,
word_interner,
*left,
*right,
*proximity,
),
WordPair::WordPrefix { left, right_prefix } => db_cache
.get_word_prefix_pair_proximity_docids(
index,
txn,
word_interner,
*left,
*right_prefix,
*proximity,
),
WordPair::WordPrefixSwapped { left_prefix, right } => db_cache
.get_prefix_word_pair_proximity_docids(
index,
txn,
word_interner,
*left_prefix,
*right,
*proximity,
),
}?;
// TODO: deserialize bitmap within a universe, and (maybe) using a bump allocator?
let bitmap = universe