Put the documents MTBL back into LMDB

We makes sure to write the documents into a file before
memory mapping it and putting it into LMDB, this way we avoid
moving it to RAM
This commit is contained in:
Clément Renault
2020-08-28 15:38:05 +02:00
parent d784d87880
commit 0a44ff86ab
10 changed files with 100 additions and 110 deletions

View File

@ -96,7 +96,7 @@ fn main() -> anyhow::Result<()> {
.open(&opt.database)?;
// Open the LMDB database.
let index = Index::new(&env, opt.database)?;
let index = Index::new(&env)?;
let rtxn = env.read_txn()?;
match opt.command {
@ -200,6 +200,11 @@ fn biggest_value_sizes(index: &Index, rtxn: &heed::RoTxn, limit: usize) -> anyho
if heap.len() > limit { heap.pop(); }
}
if let Some(documents) = index.main.get::<_, ByteSlice, ByteSlice>(rtxn, b"documents")? {
heap.push(Reverse((documents.len(), format!("documents"), main_name)));
if heap.len() > limit { heap.pop(); }
}
for result in index.word_positions.as_polymorph().iter::<_, Str, ByteSlice>(rtxn)? {
let (word, value) = result?;
heap.push(Reverse((value.len(), word.to_string(), word_positions_name)));