Come back to the old tokenizer

This commit is contained in:
Clément Renault
2020-08-30 21:50:30 +02:00
committed by Kerollmops
parent 220ba0785c
commit bad0663138
8 changed files with 45 additions and 101 deletions

View File

@ -3,7 +3,7 @@ mod node;
mod query_tokens;
mod search;
pub mod heed_codec;
pub mod lexer;
pub mod tokenizer;
use std::collections::HashMap;
use std::hash::BuildHasherDefault;