Come back to the old tokenizer

This commit is contained in:
Clément Renault
2020-08-30 21:50:30 +02:00
committed by Kerollmops
parent 220ba0785c
commit bad0663138
8 changed files with 45 additions and 101 deletions

33
Cargo.lock generated
View File

@ -6,15 +6,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccc9a9dd069569f212bc4330af9f17c4afb5e8ce185e83dbb14f1349dda18b10"
[[package]]
name = "aho-corasick"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
dependencies = [
"memchr",
]
[[package]]
name = "anyhow"
version = "1.0.31"
@ -1029,7 +1020,6 @@ dependencies = [
"structopt",
"tempfile",
"tokio",
"unicode-linebreak",
"warp",
]
@ -1624,10 +1614,7 @@ version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"thread_local 1.0.1",
]
[[package]]
@ -1851,7 +1838,7 @@ dependencies = [
"chrono",
"log 0.4.8",
"termcolor",
"thread_local 0.3.4",
"thread_local",
]
[[package]]
@ -1966,15 +1953,6 @@ dependencies = [
"unreachable",
]
[[package]]
name = "thread_local"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [
"lazy_static 1.4.0",
]
[[package]]
name = "time"
version = "0.1.43"
@ -2139,15 +2117,6 @@ dependencies = [
"matches",
]
[[package]]
name = "unicode-linebreak"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e30c7c3c3fa01e2c0da7008b57c2e5414b132a27fdf797e49e5ecbfe4f4b150"
dependencies = [
"regex",
]
[[package]]
name = "unicode-normalization"
version = "0.1.12"