Move from oxidized-mtbl to grenad

2025-07-26 16:21:07 +00:00 · 2020-10-14 09:59:27 +02:00
parent b342a86c15
commit f980422c57
3 changed files with 130 additions and 131 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -155,7 +155,7 @@ version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931"
 dependencies = [
- "lazy_static 1.4.0",
+ "lazy_static",
 "memchr",
 "regex-automata",
 "serde",
@ -251,12 +251,6 @@ dependencies = [
 "bitflags",
 ]

-[[package]]
-name = "crc32c"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ba37ef26c12988c1cee882d522d65e1d5d2ad8c3864665b88ee92767ed84c5"
-
 [[package]]
 name = "crc32fast"
 version = "1.2.0"
@ -268,9 +262,9 @@ dependencies = [

 [[package]]
 name = "criterion"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63f696897c88b57f4ffe3c69d8e1a0613c7d0e6c4833363c8560fbde9c47b966"
+checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8"
 dependencies = [
 "atty",
 "cast",
@ -278,13 +272,14 @@ dependencies = [
 "criterion-plot",
 "csv",
 "itertools",
- "lazy_static 1.4.0",
+ "lazy_static",
 "num-traits",
 "oorandom",
 "plotters",
 "rayon",
 "regex",
 "serde",
+ "serde_cbor",
 "serde_derive",
 "serde_json",
 "tinytemplate",
@ -293,9 +288,9 @@ dependencies = [

 [[package]]
 name = "criterion-plot"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddeaf7989f00f2e1d871a26a110f3ed713632feac17f65f03ca938c542618b60"
+checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d"
 dependencies = [
 "cast",
 "itertools",
@ -321,7 +316,7 @@ dependencies = [
 "autocfg 1.0.0",
 "cfg-if",
 "crossbeam-utils",
- "lazy_static 1.4.0",
+ "lazy_static",
 "maybe-uninit",
 "memoffset",
 "scopeguard",
@ -345,7 +340,7 @@ checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 dependencies = [
 "autocfg 1.0.0",
 "cfg-if",
- "lazy_static 1.4.0",
+ "lazy_static",
 ]

 [[package]]
@ -579,6 +574,20 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"

+[[package]]
+name = "grenad"
+version = "0.1.0"
+source = "git+https://github.com/Kerollmops/grenad.git?rev=a884670#a8846703ddee1f1ed86efd3168561606c244e135"
+dependencies = [
+ "byteorder",
+ "flate2",
+ "log 0.4.11",
+ "nix",
+ "snap",
+ "tempfile",
+ "zstd",
+]
+
 [[package]]
 name = "h2"
 version = "0.2.5"
@ -598,6 +607,12 @@ dependencies = [
 "tokio-util",
 ]

+[[package]]
+name = "half"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
+
 [[package]]
 name = "hashbrown"
 version = "0.8.2"
@ -835,9 +850,9 @@ dependencies = [

 [[package]]
 name = "js-sys"
-version = "0.3.40"
+version = "0.3.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177"
+checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8"
 dependencies = [
 "wasm-bindgen",
 ]
@ -852,12 +867,6 @@ dependencies = [
 "winapi-build",
 ]

-[[package]]
-name = "lazy_static"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@ -875,9 +884,9 @@ dependencies = [

 [[package]]
 name = "libc"
-version = "0.2.70"
+version = "0.2.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f"
+checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743"

 [[package]]
 name = "linked-hash-map"
@ -932,16 +941,6 @@ version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"

-[[package]]
-name = "memmap"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
-dependencies = [
- "libc",
- "winapi 0.3.8",
-]
-
 [[package]]
 name = "memoffset"
 version = "0.5.4"
@ -965,6 +964,7 @@ dependencies = [
 "flate2",
 "fst",
 "fxhash",
+ "grenad",
 "heed",
 "human_format",
 "itertools",
@ -972,10 +972,8 @@ dependencies = [
 "levenshtein_automata",
 "linked-hash-map",
 "log 0.4.11",
- "memmap",
 "near-proximity",
 "once_cell",
- "oxidized-mtbl",
 "rayon",
 "ringtail",
 "roaring",
@ -1137,6 +1135,18 @@ dependencies = [
 "winapi 0.3.8",
 ]

+[[package]]
+name = "nix"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85db2feff6bf70ebc3a4793191517d5f0331100a2f10f9bf93b5e5214f32b7b7"
+dependencies = [
+ "bitflags",
+ "cc",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "nom"
 version = "5.1.2"
@ -1200,21 +1210,6 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"

-[[package]]
-name = "oxidized-mtbl"
-version = "0.1.0"
-source = "git+https://github.com/Kerollmops/oxidized-mtbl.git?rev=9ff8082#9ff808236980305183b2440f08cf471c58e24961"
-dependencies = [
- "byteorder",
- "crc32c",
- "flate2",
- "log 0.4.11",
- "memmap",
- "snap",
- "tempfile",
- "zstd",
-]
-
 [[package]]
 name = "page_size"
 version = "0.4.2"
@ -1556,7 +1551,7 @@ dependencies = [
 "crossbeam-deque",
 "crossbeam-queue",
 "crossbeam-utils",
- "lazy_static 1.4.0",
+ "lazy_static",
 "num_cpus",
 ]

@ -1586,9 +1581,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.3.9"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
+checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b"
 dependencies = [
 "regex-syntax",
 ]
@ -1604,9 +1599,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.18"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
+checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c"

 [[package]]
 name = "remove_dir_all"
@ -1698,6 +1693,16 @@ dependencies = [
 "serde_derive",
 ]

+[[package]]
+name = "serde_cbor"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622"
+dependencies = [
+ "half",
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.110"
@ -1807,9 +1812,9 @@ dependencies = [

 [[package]]
 name = "stderrlog"
-version = "0.4.3"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e5ee9b90a5452c570a0b0ac1c99ae9498db7e56e33d74366de7f2a7add7f25"
+checksum = "b02f316286ae558d83acc93dd81eaba096e746987a7961d4a9ae026842bae67f"
 dependencies = [
 "atty",
 "chrono",
@ -1825,7 +1830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
 dependencies = [
 "clap",
- "lazy_static 1.4.0",
+ "lazy_static",
 "structopt-derive",
 ]

@ -1933,12 +1938,11 @@ dependencies = [

 [[package]]
 name = "thread_local"
-version = "0.3.4"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1697c4b57aeeb7a536b647165a2825faddffb1d3bad386d507709bd51a90bb14"
+checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
 dependencies = [
- "lazy_static 0.2.11",
- "unreachable",
+ "lazy_static",
 ]

 [[package]]
@ -1986,7 +1990,7 @@ dependencies = [
 "fnv",
 "futures-core",
 "iovec",
- "lazy_static 1.4.0",
+ "lazy_static",
 "libc",
 "memchr",
 "mio",
@ -2147,15 +2151,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"

-[[package]]
-name = "unreachable"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
-dependencies = [
- "void",
-]
-
 [[package]]
 name = "url"
 version = "2.1.1"
@ -2191,12 +2186,6 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"

-[[package]]
-name = "void"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
-
 [[package]]
 name = "walkdir"
 version = "2.3.1"
@ -2252,9 +2241,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"

 [[package]]
 name = "wasm-bindgen"
-version = "0.2.63"
+version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0"
+checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
 dependencies = [
 "cfg-if",
 "wasm-bindgen-macro",
@ -2262,12 +2251,12 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.63"
+version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101"
+checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
 dependencies = [
 "bumpalo",
- "lazy_static 1.4.0",
+ "lazy_static",
 "log 0.4.11",
 "proc-macro2",
 "quote",
@ -2277,9 +2266,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.63"
+version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3"
+checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
 dependencies = [
 "quote",
 "wasm-bindgen-macro-support",
@ -2287,9 +2276,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.63"
+version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92"
+checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
 dependencies = [
 "proc-macro2",
 "quote",
@ -2300,15 +2289,15 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.63"
+version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd"
+checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"

 [[package]]
 name = "web-sys"
-version = "0.3.40"
+version = "0.3.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17"
+checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d"
 dependencies = [
 "js-sys",
 "wasm-bindgen",