Bump the tokenizer version to v0.2.4

Cargo.lock

@@ -990,7 +990,7 @@ dependencies = [
  "jemallocator",
  "log",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
  "memmap",
  "milli",
  "once_cell",
@@ -1353,7 +1353,23 @@ dependencies = [
  "once_cell",
  "slice-group-by",
  "unicode-segmentation",
- "whatlang",
+ "whatlang 0.9.0",
+]
+
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.4"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang 0.12.0",
 ]
 
 [[package]]
@@ -1404,7 +1420,7 @@ dependencies = [
  "log",
  "logging_timer",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.4",
  "memmap",
  "obkv",
  "once_cell",
@@ -3087,6 +3103,15 @@ dependencies = [
  "hashbrown 0.7.2",
 ]
 
+[[package]]
+name = "whatlang"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703"
+dependencies = [
+ "hashbrown 0.7.2",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"

milli/Cargo.toml

@@ -19,7 +19,7 @@ heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-fe
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.4" }
 memmap = "0.7.0"
 obkv = "0.2.0"
 once_cell = "1.5.2"