feat: Use the new Tokenizer in the csv-indexer

This commit is contained in:
Clément Renault
2018-09-27 16:59:41 +02:00
parent b84be67aa2
commit 8a0c82d51e
2 changed files with 32 additions and 24 deletions

View File

@ -8,6 +8,7 @@ pub mod tokenizer;
pub use self::metadata::{Metadata, MetadataBuilder};
pub use self::rank::RankedStream;
pub use self::tokenizer::Tokenizer;
pub type DocumentId = u64;