snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-07-28 09:11:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #308

Browse Source 
308: Implement a better parallel indexer r=Kerollmops a=ManyTheFish

Rewrite the indexer:
- enhance memory consumption control
- optimize parallelism using rayon and crossbeam channel
- factorize the different parts and make new DB implementation easier
- optimize and fix prefix databases


Co-authored-by: many <maxime@meilisearch.com>
This commit is contained in:
bors[bot]
2021-09-02 15:03:52 +00:00
committed by GitHub
parent 46f7df232a 741a4444a9
commit 5cbe879325
39 changed files with 2269 additions and 1868 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
milli/Cargo.toml
View File

@ -9,12 +9,13 @@ bstr = "0.2.15"
byteorder = "1.4.2"
chrono = { version = "0.4.19", features = ["serde"] }
concat-arrays = "0.1.2"
crossbeam-channel = "0.5.1"
csv = "1.1.5"
either = "1.6.1"
flate2 = "1.0.20"
fst = "0.4.5"
fxhash = "0.2.1"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
grenad = "0.3.0"
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
human_format = "1.0.3"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }