update tokenizer dep to release

This commit is contained in:
mpostma
2020-12-03 12:34:08 +01:00
committed by many
parent a7c88c7951
commit 8b149c9aa3
2 changed files with 17 additions and 1 deletions

16
Cargo.lock generated
View File

@ -1696,6 +1696,22 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.1.0"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.1.0#32b2aee7e4a819478226eab5525232f09dd61bf5"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang",
]
[[package]]
name = "meilisearch-types"
version = "0.17.0"