Come back to the old tokenizer

This commit is contained in:
Clément Renault
2020-08-30 21:50:30 +02:00
committed by Kerollmops
parent 220ba0785c
commit bad0663138
8 changed files with 45 additions and 101 deletions

View File

@ -31,7 +31,6 @@ smallstr = "0.2.0"
smallvec = "1.4.0"
structopt = { version = "0.3.14", default-features = false }
tempfile = "3.1.0"
unicode-linebreak = "0.1.0"
# logging
log = "0.4.8"