integrate with meilisearch tokenizer

This commit is contained in:
mpostma
2020-12-23 19:09:01 +01:00
parent 7e1c94ab9c
commit 1ae761311e
10 changed files with 460 additions and 269 deletions

View File

@ -10,6 +10,7 @@ anyhow = "1.0.28"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = "0.10.5"
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
memmap = "0.7.0"
milli = { path = ".." }
once_cell = "1.4.1"