snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-20 19:56:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Improve the indexing time a little bit

Browse Source 
...by a factor of 17.6x.
This commit is contained in:
Clément Renault
2018-07-10 21:29:17 +02:00
parent a2152a4064
commit e5c54c4399
14 changed files with 497 additions and 1197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
raptor/src/rank/mod.rs
View File

@ -8,11 +8,10 @@ mod exact;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::{mem, vec};
use DocIndexMap;
use fst;
use levenshtein::Levenshtein;
use map::{OpWithStateBuilder, UnionWithState, Values};
use {Match, DocIndex, DocumentId};
use metadata::{DocIndexes, OpWithStateBuilder, UnionWithState};
use {Match, DocumentId};
use group_by::GroupByMut;
use self::sum_of_typos::sum_of_typos;
@ -117,7 +116,7 @@ impl IntoIterator for Pool {
pub enum RankedStream<'m, 'v> {
Fed {
inner: UnionWithState<'m, 'v, DocIndex, u32>,
inner: UnionWithState<'m, 'v, u32>,
automatons: Vec<Levenshtein>,
pool: Pool,
},
@ -127,11 +126,11 @@ pub enum RankedStream<'m, 'v> {
}
impl<'m, 'v> RankedStream<'m, 'v> {
pub fn new(map: &'m DocIndexMap, values: &'v Values<DocIndex>, automatons: Vec<Levenshtein>, limit: usize) -> Self {
let mut op = OpWithStateBuilder::new(values);
pub fn new(map: &'m fst::Map, indexes: &'v DocIndexes, automatons: Vec<Levenshtein>, limit: usize) -> Self {
let mut op = OpWithStateBuilder::new(indexes);
for automaton in automatons.iter().map(|l| l.dfa.clone()) {
let stream = map.as_map().search(automaton).with_state();
let stream = map.search(automaton).with_state();
op.push(stream);
}

8
raptor/src/rank/sum_of_words_position.rs
View File

@ -3,10 +3,10 @@ use Match;
use rank::{match_query_index, Document};
use group_by::GroupBy;
pub fn sum_of_words_position(lhs: &Document, rhs: &Document) -> Ordering {
let key = |matches: &[Match]| -> u32 {
GroupBy::new(matches, match_query_index).map(|m| m[0].attribute_index).sum()
};
fn key(matches: &[Match]) -> u32 {
GroupBy::new(matches, match_query_index).map(|m| m[0].attribute_index).sum()
}
pub fn sum_of_words_position(lhs: &Document, rhs: &Document) -> Ordering {
key(&lhs.matches).cmp(&key(&rhs.matches))
}