snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-29 18:04:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Loïc Lecrenier
2023-03-18 15:04:34 +01:00
parent 2853009987
commit 8b4e07e1a3
14 changed files with 367 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
milli/src/search/new/ranking_rule_graph/cheapest_paths.rs
View File

@ -4,8 +4,7 @@ use std::collections::btree_map::Entry;
use std::collections::{BTreeMap, VecDeque};
use std::ops::ControlFlow;
use super::dead_end_path_cache::DeadEndPathCache;
use super::{RankingRuleGraph, RankingRuleGraphTrait};
use super::{DeadEndsCache, RankingRuleGraph, RankingRuleGraphTrait};
use crate::search::new::interner::{Interned, MappedInterner};
use crate::search::new::query_graph::QueryNode;
use crate::search::new::small_bitmap::SmallBitmap;
@ -23,11 +22,11 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
from: Interned<QueryNode>,
cost: u16,
all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::Condition>)>, QueryNode>,
dead_end_path_cache: &mut DeadEndPathCache<G>,
dead_end_path_cache: &mut DeadEndsCache<G::Condition>,
mut visit: impl FnMut(
&[Interned<G::Condition>],
&mut Self,
&mut DeadEndPathCache<G>,
&mut DeadEndsCache<G::Condition>,
) -> Result<ControlFlow<()>>,
) -> Result<()> {
let _ = self.visit_paths_of_cost_rec(
@ -38,7 +37,7 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
&mut visit,
&mut vec![],
&mut SmallBitmap::for_interned_values_in(&self.conditions_interner),
&mut dead_end_path_cache.conditions.clone(),
&mut dead_end_path_cache.forbidden.clone(),
)?;
Ok(())
}
@ -47,11 +46,11 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
from: Interned<QueryNode>,
cost: u16,
all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::Condition>)>, QueryNode>,
dead_end_path_cache: &mut DeadEndPathCache<G>,
dead_end_path_cache: &mut DeadEndsCache<G::Condition>,
visit: &mut impl FnMut(
&[Interned<G::Condition>],
&mut Self,
&mut DeadEndPathCache<G>,
&mut DeadEndsCache<G::Condition>,
) -> Result<ControlFlow<()>>,
prev_conditions: &mut Vec<Interned<G::Condition>>,
cur_path: &mut SmallBitmap<G::Condition>,
@ -74,7 +73,6 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
ControlFlow::Continue(_) => {}
ControlFlow::Break(_) => return Ok(true),
}
true
} else {
self.visit_paths_of_cost_rec(
edge.dest_node,
@ -85,7 +83,7 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
prev_conditions,
cur_path,
forbidden_conditions,
)?
)?;
}
}
Some(condition) => {
@ -101,24 +99,20 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
}
cur_path.insert(condition);
prev_conditions.push(condition);
let mut new_forbidden_conditions = forbidden_conditions.clone();
new_forbidden_conditions
.union(dead_end_path_cache.condition_couples.get(condition));
dead_end_path_cache.prefixes.final_edges_after_prefix(
prev_conditions,
&mut |x| {
new_forbidden_conditions.insert(x);
},
);
let next_any_valid = if edge.dest_node == self.query_graph.end_node {
if let Some(next_forbidden) =
dead_end_path_cache.forbidden_conditions_after_prefix(&prev_conditions)
{
new_forbidden_conditions.union(&next_forbidden);
}
if edge.dest_node == self.query_graph.end_node {
any_valid = true;
let control_flow = visit(prev_conditions, self, dead_end_path_cache)?;
match control_flow {
ControlFlow::Continue(_) => {}
ControlFlow::Break(_) => return Ok(true),
}
true
} else {
self.visit_paths_of_cost_rec(
edge.dest_node,
@ -129,28 +123,12 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
prev_conditions,
cur_path,
&mut new_forbidden_conditions,
)?
};
)?;
}
cur_path.remove(condition);
prev_conditions.pop();
next_any_valid
}
};
any_valid |= next_any_valid;
if next_any_valid {
if dead_end_path_cache.path_is_dead_end(prev_conditions, cur_path) {
return Ok(any_valid);
}
forbidden_conditions.union(&dead_end_path_cache.conditions);
for prev_condition in prev_conditions.iter() {
forbidden_conditions
.union(dead_end_path_cache.condition_couples.get(*prev_condition));
}
dead_end_path_cache.prefixes.final_edges_after_prefix(prev_conditions, &mut |x| {
forbidden_conditions.insert(x);
});
}
}
Ok(any_valid)

14
milli/src/search/new/ranking_rule_graph/condition_docids_cache.rs
View File

@ -12,11 +12,16 @@ use crate::Result;
pub struct ConditionDocIdsCache<G: RankingRuleGraphTrait> {
// TODO: should be FxHashMap<Interned<EdgeCondition>, RoaringBitmap>
pub cache: FxHashMap<Interned<G::Condition>, RoaringBitmap>,
pub universe_length: u64,
_phantom: PhantomData<G>,
}
impl<G: RankingRuleGraphTrait> Default for ConditionDocIdsCache<G> {
fn default() -> Self {
Self { cache: Default::default(), _phantom: Default::default() }
impl<G: RankingRuleGraphTrait> ConditionDocIdsCache<G> {
pub fn new(universe: &RoaringBitmap) -> Self {
Self {
cache: Default::default(),
_phantom: Default::default(),
universe_length: universe.len(),
}
}
}
impl<G: RankingRuleGraphTrait> ConditionDocIdsCache<G> {
@ -33,6 +38,9 @@ impl<G: RankingRuleGraphTrait> ConditionDocIdsCache<G> {
universe: &RoaringBitmap,
) -> Result<&'s RoaringBitmap> {
if self.cache.contains_key(&interned_condition) {
// TODO compare length of universe compared to the one in self
// if it is smaller, then update the value
// TODO: should we update the bitmap in the cache if the new universe
// reduces it?
// TODO: maybe have a generation: u32 to track every time the universe was

157
milli/src/search/new/ranking_rule_graph/dead_end_path_cache.rs
View File

@ -1,84 +1,83 @@
use super::{path_set::PathSet, RankingRuleGraphTrait};
use crate::search::new::{
interner::{FixedSizeInterner, Interned, MappedInterner},
small_bitmap::SmallBitmap,
};
// use super::{path_set::PathSet, RankingRuleGraphTrait};
// use crate::search::new::{
// interner::{FixedSizeInterner, Interned, MappedInterner},
// small_bitmap::SmallBitmap,
// };
/// A cache which stores sufficient conditions for a path
/// to resolve to an empty set of candidates within the current
/// universe.
pub struct DeadEndPathCache<G: RankingRuleGraphTrait> {
/// The set of edge conditions that resolve to no documents.
pub conditions: SmallBitmap<G::Condition>,
/// A set of path prefixes that resolve to no documents.
pub prefixes: PathSet<G::Condition>,
/// A set of empty couples of edge conditions that resolve to no documents.
pub condition_couples: MappedInterner<SmallBitmap<G::Condition>, G::Condition>,
}
impl<G: RankingRuleGraphTrait> Clone for DeadEndPathCache<G> {
fn clone(&self) -> Self {
Self {
conditions: self.conditions.clone(),
prefixes: self.prefixes.clone(),
condition_couples: self.condition_couples.clone(),
}
}
}
// /// A cache which stores sufficient conditions for a path
// /// to resolve to an empty set of candidates within the current
// /// universe.
// pub struct DeadEndPathCache<G: RankingRuleGraphTrait> {
// /// The set of edge conditions that resolve to no documents.
// pub conditions: SmallBitmap<G::Condition>,
// /// A set of path prefixes that resolve to no documents.
// pub prefixes: PathSet<G::Condition>,
// /// A set of empty couples of edge conditions that resolve to no documents.
// pub condition_couples: MappedInterner<SmallBitmap<G::Condition>, G::Condition>,
// }
// impl<G: RankingRuleGraphTrait> Clone for DeadEndPathCache<G> {
// fn clone(&self) -> Self {
// Self {
// conditions: self.conditions.clone(),
// prefixes: self.prefixes.clone(),
// condition_couples: self.condition_couples.clone(),
// }
// }
// }
impl<G: RankingRuleGraphTrait> DeadEndPathCache<G> {
/// Create a new cache for a ranking rule graph containing at most `all_edges_len` edges.
pub fn new(all_conditions: &FixedSizeInterner<G::Condition>) -> Self {
Self {
conditions: SmallBitmap::for_interned_values_in(all_conditions),
prefixes: PathSet::default(),
condition_couples: all_conditions
.map(|_| SmallBitmap::for_interned_values_in(all_conditions)),
}
}
// impl<G: RankingRuleGraphTrait> DeadEndPathCache<G> {
// /// Create a new cache for a ranking rule graph containing at most `all_edges_len` edges.
// pub fn new(all_conditions: &FixedSizeInterner<G::Condition>) -> Self {
// Self {
// conditions: SmallBitmap::for_interned_values_in(all_conditions),
// prefixes: PathSet::default(),
// condition_couples: all_conditions
// .map(|_| SmallBitmap::for_interned_values_in(all_conditions)),
// }
// }
/// Store in the cache that every path containing the given edge resolves to no documents.
pub fn add_condition(&mut self, condition: Interned<G::Condition>) {
self.conditions.insert(condition);
self.condition_couples.get_mut(condition).clear();
self.prefixes.remove_edge(condition);
for (_, edges2) in self.condition_couples.iter_mut() {
edges2.remove(condition);
}
}
/// Store in the cache that every path containing the given prefix resolves to no documents.
pub fn add_prefix(&mut self, prefix: &[Interned<G::Condition>]) {
// TODO: typed PathSet
self.prefixes.insert(prefix.iter().copied());
}
// /// Store in the cache that every path containing the given edge resolves to no documents.
// pub fn add_condition(&mut self, condition: Interned<G::Condition>) {
// self.conditions.insert(condition);
// self.condition_couples.get_mut(condition).clear();
// self.prefixes.remove_edge(condition);
// for (_, edges2) in self.condition_couples.iter_mut() {
// edges2.remove(condition);
// }
// }
// /// Store in the cache that every path containing the given prefix resolves to no documents.
// pub fn add_prefix(&mut self, prefix: &[Interned<G::Condition>]) {
// // TODO: typed PathSet
// self.prefixes.insert(prefix.iter().copied());
// }
/// Store in the cache that every path containing the two given edges resolves to no documents.
pub fn add_condition_couple(
&mut self,
edge1: Interned<G::Condition>,
edge2: Interned<G::Condition>,
) {
self.condition_couples.get_mut(edge1).insert(edge2);
}
// /// Store in the cache that every path containing the two given edges resolves to no documents.
// pub fn add_condition_couple(
// &mut self,
// edge1: Interned<G::Condition>,
// edge2: Interned<G::Condition>,
// ) {
// self.condition_couples.get_mut(edge1).insert(edge2);
// }
/// Returns true if the cache can determine that the given path resolves to no documents.
pub fn path_is_dead_end(
&self,
path: &[Interned<G::Condition>],
path_bitmap: &SmallBitmap<G::Condition>,
) -> bool {
if path_bitmap.intersects(&self.conditions) {
return true;
}
for condition in path.iter() {
// TODO: typed path
let forbidden_other_edges = self.condition_couples.get(*condition);
if path_bitmap.intersects(forbidden_other_edges) {
return true;
}
}
if self.prefixes.contains_prefix_of_path(path) {
return true;
}
false
}
}
// /// Returns true if the cache can determine that the given path resolves to no documents.
// pub fn path_is_dead_end(
// &self,
// path: &[Interned<G::Condition>],
// path_bitmap: &SmallBitmap<G::Condition>,
// ) -> bool {
// if path_bitmap.intersects(&self.conditions) {
// return true;
// }
// for condition in path.iter() {
// let forbidden_other_edges = self.condition_couples.get(*condition);
// if path_bitmap.intersects(forbidden_other_edges) {
// return true;
// }
// }
// if self.prefixes.contains_prefix_of_path(path) {
// return true;
// }
// false
// }
// }

5
milli/src/search/new/ranking_rule_graph/mod.rs
View File

@ -20,7 +20,8 @@ use std::collections::HashSet;
use std::hash::Hash;
pub use condition_docids_cache::ConditionDocIdsCache;
pub use dead_end_path_cache::DeadEndPathCache;
// pub use dead_end_path_cache::DeadEndPathCache;
pub use path_set::DeadEndsCache;
pub use proximity::{ProximityCondition, ProximityGraph};
use roaring::RoaringBitmap;
pub use typo::{TypoCondition, TypoGraph};
@ -113,7 +114,7 @@ pub trait RankingRuleGraphTrait: Sized {
fn log_state(
graph: &RankingRuleGraph<Self>,
paths: &[Vec<Interned<Self::Condition>>],
dead_end_path_cache: &DeadEndPathCache<Self>,
dead_end_path_cache: &DeadEndsCache<Self::Condition>,
universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<Self::Condition>)>, QueryNode>,
cost: u16,

223
milli/src/search/new/ranking_rule_graph/path_set.rs
View File

@ -2,104 +2,165 @@
// For the empty_prefixes field in the EmptyPathsCache only :/
// but it could be used for more, like efficient computing of a set of paths
use crate::search::new::interner::Interned;
use crate::search::new::{
interner::{FixedSizeInterner, Interned},
small_bitmap::SmallBitmap,
};
/// A set of `Vec<Interned<T>>` implemented as a prefix tree.
pub struct PathSet<T> {
pub struct DeadEndsCache<T> {
nodes: Vec<(Interned<T>, Self)>,
is_end: bool,
pub forbidden: SmallBitmap<T>,
}
impl<T> Clone for PathSet<T> {
fn clone(&self) -> Self {
Self { nodes: self.nodes.clone(), is_end: self.is_end }
impl<T> DeadEndsCache<T> {
pub fn new(for_interner: &FixedSizeInterner<T>) -> Self {
Self { nodes: vec![], forbidden: SmallBitmap::for_interned_values_in(for_interner) }
}
}
impl<T> std::fmt::Debug for PathSet<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PathSet").field("nodes", &self.nodes).field("is_end", &self.is_end).finish()
pub fn forbid_condition(&mut self, condition: Interned<T>) {
self.forbidden.insert(condition);
}
}
impl<T> Default for PathSet<T> {
fn default() -> Self {
Self { nodes: Default::default(), is_end: Default::default() }
}
}
impl<T> PathSet<T> {
pub fn insert(&mut self, mut edges: impl Iterator<Item = Interned<T>>) {
match edges.next() {
None => {
self.is_end = true;
}
Some(first_edge) => {
for (edge, next_node) in &mut self.nodes {
if edge == &first_edge {
return next_node.insert(edges);
}
}
let mut rest = PathSet::default();
rest.insert(edges);
self.nodes.push((first_edge, rest));
fn advance(&mut self, condition: Interned<T>) -> Option<&mut Self> {
for (e, next_node) in &mut self.nodes {
if condition == *e {
return Some(next_node);
}
}
None
}
pub fn remove_edge(&mut self, forbidden_edge: Interned<T>) {
let mut i = 0;
while i < self.nodes.len() {
let should_remove = if self.nodes[i].0 == forbidden_edge {
true
} else if !self.nodes[i].1.nodes.is_empty() {
self.nodes[i].1.remove_edge(forbidden_edge);
self.nodes[i].1.nodes.is_empty()
pub fn forbidden_conditions_after_prefix(
&mut self,
mut prefix: &[Interned<T>],
) -> Option<SmallBitmap<T>> {
let mut cursor = self;
for c in prefix.iter() {
if let Some(next) = cursor.advance(*c) {
cursor = next;
} else {
false
};
if should_remove {
self.nodes.remove(i);
} else {
i += 1;
return None;
}
}
Some(cursor.forbidden.clone())
}
pub fn final_edges_after_prefix(
&self,
prefix: &[Interned<T>],
visit: &mut impl FnMut(Interned<T>),
pub fn forbid_condition_after_prefix(
&mut self,
mut prefix: impl Iterator<Item = Interned<T>>,
forbidden: Interned<T>,
) {
let [first_edge, remaining_prefix @ ..] = prefix else {
for node in self.nodes.iter() {
if node.1.is_end {
visit(node.0)
}
match prefix.next() {
None => {
self.forbidden.insert(forbidden);
}
return
};
for (edge, rest) in self.nodes.iter() {
if edge == first_edge {
return rest.final_edges_after_prefix(remaining_prefix, visit);
}
}
}
pub fn contains_prefix_of_path(&self, path: &[Interned<T>]) -> bool {
if self.is_end {
return true;
}
match path {
[] => false,
[first_edge, remaining_path @ ..] => {
for (edge, rest) in self.nodes.iter() {
if edge == first_edge {
return rest.contains_prefix_of_path(remaining_path);
Some(first_condition) => {
for (condition, next_node) in &mut self.nodes {
if condition == &first_condition {
return next_node.forbid_condition_after_prefix(prefix, forbidden);
}
}
false
let mut rest = DeadEndsCache {
nodes: vec![],
forbidden: SmallBitmap::new(self.forbidden.universe_length()),
};
rest.forbid_condition_after_prefix(prefix, forbidden);
self.nodes.push((first_condition, rest));
}
}
}
}
// /// A set of `Vec<Interned<T>>` implemented as a prefix tree.
// pub struct PathSet<T> {
// nodes: Vec<(Interned<T>, Self)>,
// is_end: bool,
// }
// impl<T> Clone for PathSet<T> {
// fn clone(&self) -> Self {
// Self { nodes: self.nodes.clone(), is_end: self.is_end }
// }
// }
// impl<T> std::fmt::Debug for PathSet<T> {
// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// f.debug_struct("PathSet").field("nodes", &self.nodes).field("is_end", &self.is_end).finish()
// }
// }
// impl<T> Default for PathSet<T> {
// fn default() -> Self {
// Self { nodes: Default::default(), is_end: Default::default() }
// }
// }
// impl<T> PathSet<T> {
// pub fn insert(&mut self, mut conditions: impl Iterator<Item = Interned<T>>) {
// match conditions.next() {
// None => {
// self.is_end = true;
// }
// Some(first_condition) => {
// for (condition, next_node) in &mut self.nodes {
// if condition == &first_condition {
// return next_node.insert(conditions);
// }
// }
// let mut rest = PathSet::default();
// rest.insert(conditions);
// self.nodes.push((first_condition, rest));
// }
// }
// }
// pub fn remove_condition(&mut self, forbidden_condition: Interned<T>) {
// let mut i = 0;
// while i < self.nodes.len() {
// let should_remove = if self.nodes[i].0 == forbidden_condition {
// true
// } else if !self.nodes[i].1.nodes.is_empty() {
// self.nodes[i].1.remove_condition(forbidden_condition);
// self.nodes[i].1.nodes.is_empty()
// } else {
// false
// };
// if should_remove {
// self.nodes.remove(i);
// } else {
// i += 1;
// }
// }
// }
// pub fn final_conditions_after_prefix(
// &self,
// prefix: &[Interned<T>],
// visit: &mut impl FnMut(Interned<T>),
// ) {
// let [first_condition, remaining_prefix @ ..] = prefix else {
// for node in self.nodes.iter() {
// if node.1.is_end {
// visit(node.0)
// }
// }
// return
// };
// for (condition, rest) in self.nodes.iter() {
// if condition == first_condition {
// return rest.final_conditions_after_prefix(remaining_prefix, visit);
// }
// }
// }
// pub fn contains_prefix_of_path(&self, path: &[Interned<T>]) -> bool {
// if self.is_end {
// return true;
// }
// match path {
// [] => false,
// [first_condition, remaining_path @ ..] => {
// for (condition, rest) in self.nodes.iter() {
// if condition == first_condition {
// return rest.contains_prefix_of_path(remaining_path);
// }
// }
// false
// }
// }
// }
// }

5
milli/src/search/new/ranking_rule_graph/proximity/mod.rs
View File

@ -6,8 +6,7 @@ use std::iter::FromIterator;
use roaring::RoaringBitmap;
use super::dead_end_path_cache::DeadEndPathCache;
use super::{RankingRuleGraph, RankingRuleGraphTrait};
use super::{RankingRuleGraph, RankingRuleGraphTrait, DeadEndsCache};
use crate::search::new::interner::{DedupInterner, Interned, MappedInterner};
use crate::search::new::logger::SearchLogger;
use crate::search::new::query_term::{Phrase, QueryTerm};
@ -67,7 +66,7 @@ impl RankingRuleGraphTrait for ProximityGraph {
fn log_state(
graph: &RankingRuleGraph<Self>,
paths: &[Vec<Interned<ProximityCondition>>],
dead_end_path_cache: &DeadEndPathCache<Self>,
dead_end_path_cache: &DeadEndsCache<Self::Condition>,
universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,
cost: u16,

5
milli/src/search/new/ranking_rule_graph/typo/mod.rs
View File

@ -1,7 +1,6 @@
use roaring::RoaringBitmap;
use super::dead_end_path_cache::DeadEndPathCache;
use super::{RankingRuleGraph, RankingRuleGraphTrait};
use super::{RankingRuleGraph, RankingRuleGraphTrait, DeadEndsCache};
use crate::search::new::interner::{DedupInterner, Interned, MappedInterner};
use crate::search::new::logger::SearchLogger;
use crate::search::new::query_graph::QueryNodeData;
@ -137,7 +136,7 @@ impl RankingRuleGraphTrait for TypoGraph {
fn log_state(
graph: &RankingRuleGraph<Self>,
paths: &[Vec<Interned<TypoCondition>>],
dead_end_path_cache: &DeadEndPathCache<Self>,
dead_end_path_cache: &DeadEndsCache<TypoCondition>,
universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoCondition>)>, QueryNode>,
cost: u16,