snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-27 08:41:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Come back to the old tokenizer

Browse Source
This commit is contained in:
Clément Renault
2020-08-30 21:50:30 +02:00
committed by Kerollmops
parent 220ba0785c
commit bad0663138
8 changed files with 45 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/query_tokens.rs
View File

@ -1,5 +1,4 @@
use std::{mem, str};
use unicode_linebreak::{break_property, BreakClass};
use QueryToken::{Quoted, Free};
@ -69,12 +68,6 @@ impl<'a> Iterator for QueryTokens<'a> {
},
State::Fused => return None,
}
} else if break_property(c as u32) == BreakClass::Ideographic {
match self.state.replace_by(State::Free(afteri)) {
State::Quoted(s) => return Some(Quoted(&self.string[s..afteri])),
State::Free(s) => return Some(Free(&self.string[s..afteri])),
_ => self.state = State::Free(afteri),
}
} else if !self.state.is_quoted() && !c.is_alphanumeric() {
match self.state.replace_by(State::Free(afteri)) {
State::Free(s) if i > s => return Some(Free(&self.string[s..i])),