Introduce MTBL parallel merging before LMDB writing

This commit is contained in:
Kerollmops
2020-05-31 14:20:17 +02:00
parent 6762c2d08f
commit 24587148fd
3 changed files with 372 additions and 114 deletions

View File

@ -8,16 +8,20 @@ edition = "2018"
anyhow = "1.0.28"
bitpacking = "0.8.2"
byteorder = "1.3.4"
roaring = "0.5.2"
csv = "1.1.3"
fst = "0.4.3"
fxhash = "0.2.1"
heed = { version = "0.8.0", default-features = false, features = ["lmdb"] }
jemallocator = "0.3.2"
memmap = "0.7.0"
oxidized-mtbl = { git = "https://github.com/Kerollmops/oxidized-mtbl.git", rev = "8918476" }
rayon = "1.3.0"
roaring = "0.5.2"
slice-group-by = "0.2.6"
smallstr = "0.2.0"
smallvec = "1.4.0"
structopt = { version = "0.3.14", default-features = false }
tempfile = "3.1.0"
[profile.release]
debug = true