Make the tokenizer understand strange whitespaces/quotes

This commit is contained in:
Clément Renault
2019-11-04 16:10:13 +01:00
parent ccded7b429
commit de2b8672d4
2 changed files with 6 additions and 1 deletions

View File

@ -5,4 +5,5 @@ authors = ["Kerollmops <renault.cle@gmail.com>"]
edition = "2018"
[dependencies]
deunicode = "1.0.0"
slice-group-by = "0.2.4"