snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-27 16:51:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Index and store/serialize attributes while creating the update

Browse Source
This commit is contained in:
Clément Renault
2018-12-07 11:32:27 +01:00
parent 442834c28f
commit 731ed11153
7 changed files with 240 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/tokenizer/mod.rs
View File

@ -1,52 +1,6 @@
use std::mem;
use self::Separator::*;
struct MegaTokenizer<I> {
strings: I,
}
impl From<String> for MegaTokenizer<Option<String>> {
fn from(string: String) -> Self {
MegaTokenizer { strings: Some(string) }
}
}
impl From<Vec<String>> for MegaTokenizer<Vec<String>> {
fn from(strings: Vec<String>) -> Self {
MegaTokenizer { strings }
}
}
impl<I> Iterator for MegaTokenizer<I> {
type Item = (usize, String);
fn next(&mut self) -> Option<Self::Item> {
unimplemented!()
}
}
#[test]
fn xxx() {
let s1 = "hello world!";
let mut s1 = MegaTokenizer::from(s1.to_owned());
assert_eq!(s1.next(), Some((0, "hello".into())));
assert_eq!(s1.next(), Some((1, "world".into())));
assert_eq!(s1.next(), None);
let v1 = vec!["Vin Diesel".to_owned(), "Quentin Tarantino".to_owned()];
let mut v1 = MegaTokenizer::from(v1);
assert_eq!(v1.next(), Some((0, "Vin".into())));
assert_eq!(v1.next(), Some((1, "Diesel".into())));
assert_eq!(v1.next(), Some((8, "Quentin".into())));
assert_eq!(v1.next(), Some((9, "Tarantino".into())));
assert_eq!(v1.next(), None);
}
pub trait TokenizerBuilder {
fn build<'a>(&self, text: &'a str) -> Box<Iterator<Item=(usize, &'a str)> + 'a>;
}