integrate with meilisearch tokenizer

This commit is contained in:
mpostma
2020-12-23 19:09:01 +01:00
parent 7e1c94ab9c
commit 1ae761311e
10 changed files with 460 additions and 269 deletions

View File

@ -13,7 +13,7 @@ crossbeam-channel = "0.5.0"
csv = "1.1.3"
either = "1.6.1"
flate2 = "1.0.17"
fst = "0.4.4"
fst = "0.4.5"
fxhash = "0.2.1"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { version = "0.10.5", default-features = false, features = ["lmdb", "sync-read-txn"] }
@ -21,6 +21,7 @@ human_format = "1.0.3"
jemallocator = "0.3.2"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
linked-hash-map = "0.5.3"
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
memmap = "0.7.0"
near-proximity = { git = "https://github.com/Kerollmops/plane-sweep-proximity", rev = "6608205" }
num-traits = "0.2.14"
@ -56,7 +57,7 @@ criterion = "0.3.3"
maplit = "1.0.2"
[build-dependencies]
fst = "0.4.4"
fst = "0.4.5"
[features]
default = []