snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-28 01:01:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite cheapest path algorithm and empty path cache

Browse Source 
It is now much simpler and has much better performance.
This commit is contained in:
Loïc Lecrenier
2023-03-02 21:27:42 +01:00
parent caa1e1b923
commit c27ea2677f
14 changed files with 782 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
milli/src/search/new/graph_based_ranking_rule.rs
View File

@ -3,12 +3,11 @@ use roaring::RoaringBitmap;
use super::db_cache::DatabaseCache;
use super::logger::SearchLogger;
use super::ranking_rule_graph::cheapest_paths::KCheapestPathsState;
use super::ranking_rule_graph::edge_docids_cache::EdgeDocidsCache;
use super::ranking_rule_graph::empty_paths_cache::EmptyPathsCache;
use super::ranking_rule_graph::paths_map::PathsMap;
use super::ranking_rule_graph::{RankingRuleGraph, RankingRuleGraphTrait};
use super::{QueryGraph, RankingRule, RankingRuleOutput};
use super::{BitmapOrAllRef, QueryGraph, RankingRule, RankingRuleOutput};
use crate::{Index, Result};
@ -24,9 +23,40 @@ impl<G: RankingRuleGraphTrait> GraphBasedRankingRule<G> {
pub struct GraphBasedRankingRuleState<G: RankingRuleGraphTrait> {
graph: RankingRuleGraph<G>,
cheapest_paths_state: Option<KCheapestPathsState>,
edge_docids_cache: EdgeDocidsCache<G>,
empty_paths_cache: EmptyPathsCache,
all_distances: Vec<Vec<u64>>,
cur_distance_idx: usize,
}
fn remove_empty_edges<'transaction, G: RankingRuleGraphTrait>(
graph: &mut RankingRuleGraph<G>,
edge_docids_cache: &mut EdgeDocidsCache<G>,
index: &Index,
txn: &'transaction RoTxn,
db_cache: &mut DatabaseCache<'transaction>,
universe: &RoaringBitmap,
empty_paths_cache: &mut EmptyPathsCache,
) -> Result<()> {
for edge_index in 0..graph.all_edges.len() as u32 {
if graph.all_edges[edge_index as usize].is_none() {
continue;
}
let docids = edge_docids_cache
.get_edge_docids(index, txn, db_cache, edge_index, &*graph, universe)?;
match docids {
BitmapOrAllRef::Bitmap(bitmap) => {
if bitmap.is_disjoint(universe) {
graph.remove_edge(edge_index);
empty_paths_cache.forbid_edge(edge_index);
edge_docids_cache.cache.remove(&edge_index);
continue;
}
}
BitmapOrAllRef::All => continue,
}
}
Ok(())
}
impl<'transaction, G: RankingRuleGraphTrait> RankingRule<'transaction, QueryGraph>
@ -41,18 +71,31 @@ impl<'transaction, G: RankingRuleGraphTrait> RankingRule<'transaction, QueryGrap
txn: &'transaction RoTxn,
db_cache: &mut DatabaseCache<'transaction>,
_logger: &mut dyn SearchLogger<QueryGraph>,
_universe: &RoaringBitmap,
universe: &RoaringBitmap,
query_graph: &QueryGraph,
) -> Result<()> {
// TODO: update old state instead of starting from scratch
let graph = RankingRuleGraph::build(index, txn, db_cache, query_graph.clone())?;
let mut graph = RankingRuleGraph::build(index, txn, db_cache, query_graph.clone())?;
let mut edge_docids_cache = EdgeDocidsCache::default();
let mut empty_paths_cache = EmptyPathsCache::new(graph.all_edges.len());
remove_empty_edges(
&mut graph,
&mut edge_docids_cache,
index,
txn,
db_cache,
universe,
&mut empty_paths_cache,
)?;
let all_distances = graph.initialize_distances_cheapest();
let cheapest_paths_state = KCheapestPathsState::new(&graph);
let state = GraphBasedRankingRuleState {
graph,
cheapest_paths_state,
edge_docids_cache: <_>::default(),
empty_paths_cache: <_>::default(),
edge_docids_cache,
empty_paths_cache,
all_distances,
cur_distance_idx: 0,
};
self.state = Some(state);
@ -70,34 +113,42 @@ impl<'transaction, G: RankingRuleGraphTrait> RankingRule<'transaction, QueryGrap
) -> Result<Option<RankingRuleOutput<QueryGraph>>> {
assert!(universe.len() > 1);
let mut state = self.state.take().unwrap();
if state.cheapest_paths_state.is_none() {
remove_empty_edges(
&mut state.graph,
&mut state.edge_docids_cache,
index,
txn,
db_cache,
universe,
&mut state.empty_paths_cache,
)?;
if state.cur_distance_idx
>= state.all_distances[state.graph.query_graph.root_node as usize].len()
{
self.state = None;
return Ok(None);
}
let cost =
state.all_distances[state.graph.query_graph.root_node as usize][state.cur_distance_idx];
state.cur_distance_idx += 1;
let mut paths = PathsMap::default();
let paths = state.graph.paths_of_cost(
state.graph.query_graph.root_node as usize,
cost,
&state.all_distances,
&state.empty_paths_cache,
);
while paths.is_empty() {
let Some(cheapest_paths_state) = state.cheapest_paths_state.take() else {
break;
};
if let Some(next_cheapest_paths_state) = cheapest_paths_state
.compute_paths_of_next_lowest_cost(
&mut state.graph,
&state.empty_paths_cache,
&mut paths,
)
{
state.cheapest_paths_state = Some(next_cheapest_paths_state);
} else {
break;
}
}
if paths.is_empty() && state.cheapest_paths_state.is_none() {
return Ok(None);
}
G::log_state(&state.graph, &paths, &state.empty_paths_cache, logger);
G::log_state(
&state.graph,
&paths,
&state.empty_paths_cache,
universe,
&state.all_distances,
cost,
logger,
);
let bucket = state.graph.resolve_paths(
index,