mirror of
https://github.com/meilisearch/meilisearch.git
synced 2025-07-31 02:40:01 +00:00
Add documentation
This commit is contained in:
Loïc Lecrenier
@ -3,6 +3,17 @@ use super::small_bitmap::SmallBitmap;
|
||||
use super::SearchContext;
|
||||
use crate::Result;
|
||||
|
||||
const QUERY_GRAPH_NODE_LENGTH_LIMIT: u16 = 64;
|
||||
|
||||
/// A node of the [`QueryGraph`].
|
||||
///
|
||||
/// There are four types of nodes:
|
||||
/// 1. `Start` : unique, represents the start of the query
|
||||
/// 2. `End` : unique, represents the end of a query
|
||||
/// 3. `Deleted` : represents a node that was deleted.
|
||||
/// All deleted nodes are unreachable from the start node.
|
||||
/// 4. `Term` is a regular node representing a word or combination of words
|
||||
/// from the user query.
|
||||
#[derive(Clone)]
|
||||
pub enum QueryNode {
|
||||
Term(LocatedQueryTerm),
|
||||
@ -11,34 +22,84 @@ pub enum QueryNode {
|
||||
End,
|
||||
}
|
||||
|
||||
/// The edges associated with a node in the query graph.
|
||||
#[derive(Clone)]
|
||||
pub struct Edges {
|
||||
// TODO: use a tiny bitset instead, something like a simple Vec<u8> where most queries will see a vector of one element
|
||||
/// Set of nodes which have an edge going to the current node
|
||||
pub predecessors: SmallBitmap,
|
||||
/// Set of nodes which are reached by an edge from the current node
|
||||
pub successors: SmallBitmap,
|
||||
}
|
||||
|
||||
/**
|
||||
A graph representing all the ways to interpret the user's search query.
|
||||
|
||||
## Important
|
||||
At the moment, a query graph has a hardcoded limit of [`QUERY_GRAPH_NODE_LENGTH_LIMIT`] nodes.
|
||||
|
||||
## Example 1
|
||||
For the search query `sunflower`, we need to register the following things:
|
||||
- we need to look for the exact word `sunflower`
|
||||
- but also any word which is 1 or 2 typos apart from `sunflower`
|
||||
- and every word that contains the prefix `sunflower`
|
||||
- and also the couple of adjacent words `sun flower`
|
||||
- as well as all the user-defined synonyms of `sunflower`
|
||||
|
||||
All these derivations of a word will be stored in [`WordDerivations`].
|
||||
|
||||
## Example 2:
|
||||
For the search query `summer house by`.
|
||||
|
||||
We also look for all word derivations of each term. And we also need to consider
|
||||
the potential n-grams `summerhouse`, `summerhouseby`, and `houseby`.
|
||||
Furthermore, we need to know which words these ngrams replace. This is done by creating the
|
||||
following graph, where each node also contains a list of derivations:
|
||||
```txt
|
||||
┌───────┐
|
||||
┌─│houseby│─────────┐
|
||||
│ └───────┘ │
|
||||
┌───────┐ ┌───────┐ │ ┌───────┐ ┌────┐ │ ┌───────┐
|
||||
│ START │─┬─│summer │─┴─│ house │┌─│ by │─┼─│ END │
|
||||
└───────┘ │ └───────┘ └───────┘│ └────┘ │ └───────┘
|
||||
│ ┌────────────┐ │ │
|
||||
├─│summerhouse │───────┘ │
|
||||
│ └────────────┘ │
|
||||
│ ┌─────────────┐ │
|
||||
└─────────│summerhouseby│───────┘
|
||||
└─────────────┘
|
||||
```
|
||||
Note also that each node has a range of positions associated with it,
|
||||
such that `summer` is known to be a word at the positions `0..=0` and `houseby`
|
||||
is registered with the positions `1..=2`. When two nodes are connected by an edge,
|
||||
it means that they are potentially next to each other in the user's search query
|
||||
(depending on the [`TermsMatchingStrategy`](crate::search::TermsMatchingStrategy)
|
||||
and the transformations that were done on the query graph).
|
||||
*/
|
||||
#[derive(Clone)]
|
||||
pub struct QueryGraph {
|
||||
/// The index of the start node within `self.nodes`
|
||||
pub root_node: u16,
|
||||
/// The index of the end node within `self.nodes`
|
||||
pub end_node: u16,
|
||||
/// The list of all query nodes
|
||||
pub nodes: Vec<QueryNode>,
|
||||
/// The list of all node edges
|
||||
pub edges: Vec<Edges>,
|
||||
}
|
||||
|
||||
fn _assert_sizes() {
|
||||
// TODO: QueryNodes are too big now, 88B is a bit too big
|
||||
let _: [u8; 88] = [0; std::mem::size_of::<QueryNode>()];
|
||||
let _: [u8; 32] = [0; std::mem::size_of::<Edges>()];
|
||||
}
|
||||
|
||||
impl Default for QueryGraph {
|
||||
/// Create a new QueryGraph with two disconnected nodes: the root and end nodes.
|
||||
fn default() -> Self {
|
||||
let nodes = vec![QueryNode::Start, QueryNode::End];
|
||||
let edges = vec![
|
||||
Edges { predecessors: SmallBitmap::new(64), successors: SmallBitmap::new(64) },
|
||||
Edges { predecessors: SmallBitmap::new(64), successors: SmallBitmap::new(64) },
|
||||
Edges {
|
||||
predecessors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
successors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
},
|
||||
Edges {
|
||||
predecessors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
successors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
},
|
||||
];
|
||||
|
||||
Self { root_node: 0, end_node: 1, nodes, edges }
|
||||
@ -46,33 +107,31 @@ impl Default for QueryGraph {
|
||||
}
|
||||
|
||||
impl QueryGraph {
|
||||
/// Connect all the given predecessor nodes to the given successor node
|
||||
fn connect_to_node(&mut self, from_nodes: &[u16], to_node: u16) {
|
||||
for &from_node in from_nodes {
|
||||
self.edges[from_node as usize].successors.insert(to_node);
|
||||
self.edges[to_node as usize].predecessors.insert(from_node);
|
||||
}
|
||||
}
|
||||
/// Add the given node to the graph and connect it to all the given predecessor nodes
|
||||
fn add_node(&mut self, from_nodes: &[u16], node: QueryNode) -> u16 {
|
||||
let new_node_idx = self.nodes.len() as u16;
|
||||
assert!(new_node_idx <= QUERY_GRAPH_NODE_LENGTH_LIMIT);
|
||||
self.nodes.push(node);
|
||||
self.edges.push(Edges {
|
||||
predecessors: SmallBitmap::from_array(from_nodes, 64),
|
||||
successors: SmallBitmap::new(64),
|
||||
predecessors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
successors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
});
|
||||
for from_node in from_nodes {
|
||||
self.edges[*from_node as usize].successors.insert(new_node_idx);
|
||||
}
|
||||
self.connect_to_node(from_nodes, new_node_idx);
|
||||
|
||||
new_node_idx
|
||||
}
|
||||
}
|
||||
|
||||
impl QueryGraph {
|
||||
// TODO: return the list of all matching words here as well
|
||||
/// Build the query graph from the parsed user search query.
|
||||
pub fn from_query(ctx: &mut SearchContext, terms: Vec<LocatedQueryTerm>) -> Result<QueryGraph> {
|
||||
// TODO: maybe empty nodes should not be removed here, to compute
|
||||
// the score of the `words` ranking rule correctly
|
||||
// it is very easy to traverse the graph and remove afterwards anyway
|
||||
// Still, I'm keeping this here as a demo
|
||||
let mut empty_nodes = vec![];
|
||||
|
||||
let word_set = ctx.index.words_fst(ctx.txn)?;
|
||||
@ -81,7 +140,6 @@ impl QueryGraph {
|
||||
let (mut prev2, mut prev1, mut prev0): (Vec<u16>, Vec<u16>, Vec<u16>) =
|
||||
(vec![], vec![], vec![graph.root_node]);
|
||||
|
||||
// TODO: split words / synonyms
|
||||
for length in 1..=terms.len() {
|
||||
let query = &terms[..length];
|
||||
|
||||
@ -156,6 +214,8 @@ impl QueryGraph {
|
||||
|
||||
Ok(graph)
|
||||
}
|
||||
|
||||
/// Remove the given nodes and all their edges from the query graph.
|
||||
pub fn remove_nodes(&mut self, nodes: &[u16]) {
|
||||
for &node in nodes {
|
||||
self.nodes[node as usize] = QueryNode::Deleted;
|
||||
@ -166,10 +226,13 @@ impl QueryGraph {
|
||||
for succ in edges.successors.iter() {
|
||||
self.edges[succ as usize].predecessors.remove(node);
|
||||
}
|
||||
self.edges[node as usize] =
|
||||
Edges { predecessors: SmallBitmap::new(64), successors: SmallBitmap::new(64) };
|
||||
self.edges[node as usize] = Edges {
|
||||
predecessors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
successors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
};
|
||||
}
|
||||
}
|
||||
/// Remove the given nodes, connecting all their predecessors to all their successors.
|
||||
pub fn remove_nodes_keep_edges(&mut self, nodes: &[u16]) {
|
||||
for &node in nodes {
|
||||
self.nodes[node as usize] = QueryNode::Deleted;
|
||||
@ -182,11 +245,17 @@ impl QueryGraph {
|
||||
self.edges[succ as usize].predecessors.remove(node);
|
||||
self.edges[succ as usize].predecessors.union(&edges.predecessors);
|
||||
}
|
||||
self.edges[node as usize] =
|
||||
Edges { predecessors: SmallBitmap::new(64), successors: SmallBitmap::new(64) };
|
||||
self.edges[node as usize] = Edges {
|
||||
predecessors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
successors: SmallBitmap::new(QUERY_GRAPH_NODE_LENGTH_LIMIT),
|
||||
};
|
||||
}
|
||||
}
|
||||
pub fn remove_words_at_position(&mut self, position: i8) -> bool {
|
||||
|
||||
/// Remove all the nodes that correspond to a word starting at the given position, and connect
|
||||
/// the predecessors of these nodes to their successors.
|
||||
/// Return `true` if any node was removed.
|
||||
pub fn remove_words_starting_at_position(&mut self, position: i8) -> bool {
|
||||
let mut nodes_to_remove_keeping_edges = vec![];
|
||||
for (node_idx, node) in self.nodes.iter().enumerate() {
|
||||
let node_idx = node_idx as u16;
|
||||
@ -202,14 +271,15 @@ impl QueryGraph {
|
||||
!nodes_to_remove_keeping_edges.is_empty()
|
||||
}
|
||||
|
||||
/// Simplify the query graph by removing all nodes that are disconnected from
|
||||
/// the start or end nodes.
|
||||
fn simplify(&mut self) {
|
||||
loop {
|
||||
let mut nodes_to_remove = vec![];
|
||||
for (node_idx, node) in self.nodes.iter().enumerate() {
|
||||
if (!matches!(node, QueryNode::End | QueryNode::Deleted)
|
||||
&& self.edges[node_idx].successors.is_empty())
|
||||
|| (!matches!(node, QueryNode::Start | QueryNode::Deleted)
|
||||
&& self.edges[node_idx].predecessors.is_empty())
|
||||
if !matches!(node, QueryNode::End | QueryNode::Deleted)
|
||||
&& (self.edges[node_idx].successors.is_empty()
|
||||
|| self.edges[node_idx].predecessors.is_empty())
|
||||
{
|
||||
nodes_to_remove.push(node_idx as u16);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user