Move the documents MTBL database inside the Index

src/bin/indexer.rs

@@ -424,7 +424,7 @@ fn main() -> anyhow::Result<()> {
         .max_dbs(10)
         .open(&opt.database)?;
 
-    let index = Index::new(&env)?;
+    let mut index = Index::new(&env, &opt.database)?;
 
     let documents_path = opt.database.join("documents.mtbl");
     let num_threads = rayon::current_num_threads();
@@ -499,13 +499,12 @@ fn main() -> anyhow::Result<()> {
         let mut builder = Merger::builder(docs_merge);
         builder.extend(docs_stores);
         builder.build().write_into(&mut writer)?;
-        Ok(writer.into_inner()?) as anyhow::Result<_>
+        Ok(writer.finish()?) as anyhow::Result<_>
     });
 
-    let file = lmdb.and(mtbl)?;
-    let mmap = unsafe { Mmap::map(&file)? };
-    let documents = Reader::new(mmap)?;
-    let count = documents.metadata().count_entries;
+    lmdb.and(mtbl)?;
+    index.refresh_documents()?;
+    let count = index.number_of_documents();
 
     debug!("Wrote {} documents into LMDB", count);
 

src/bin/search.rs

@@ -1,4 +1,3 @@
-use std::fs::File;
 use std::io::{self, Write, BufRead};
 use std::iter::once;
 use std::path::PathBuf;
@@ -7,7 +6,6 @@ use std::time::Instant;
 use heed::EnvOpenOptions;
 use log::debug;
 use milli::Index;
-use oxidized_mtbl::Reader;
 use structopt::StructOpt;
 
 #[cfg(target_os = "linux")]
@@ -47,14 +45,7 @@ fn main() -> anyhow::Result<()> {
         .open(&opt.database)?;
 
     // Open the LMDB database.
-    let index = Index::new(&env)?;
-
-    // Open the documents MTBL database.
-    let path = opt.database.join("documents.mtbl");
-    let file = File::open(path)?;
-    let mmap = unsafe { memmap::Mmap::map(&file)? };
-    let documents = Reader::new(mmap.as_ref())?;
-
+    let index = Index::new(&env, opt.database)?;
     let rtxn = env.read_txn()?;
 
     let stdin = io::stdin();
@@ -72,15 +63,13 @@ fn main() -> anyhow::Result<()> {
             Some(headers) => headers,
             None => return Ok(()),
         };
+        let documents = index.documents(documents_ids.iter().cloned())?;
 
         let mut stdout = io::stdout();
         stdout.write_all(&headers)?;
 
-        for id in &documents_ids {
-            let id_bytes = id.to_be_bytes();
-            if let Some(content) = documents.clone().get(&id_bytes)? {
-                stdout.write_all(content.as_ref())?;
-            }
+        for (_id, content) in documents {
+            stdout.write_all(&content)?;
         }
 
         debug!("Took {:.02?} to find {} documents", before.elapsed(), documents_ids.len());

src/bin/serve.rs

@@ -9,7 +9,6 @@ use std::time::Instant;
 
 use askama_warp::Template;
 use heed::EnvOpenOptions;
-use oxidized_mtbl::Reader;
 use serde::Deserialize;
 use slice_group_by::StrGroupBy;
 use structopt::StructOpt;
@@ -99,22 +98,13 @@ async fn main() -> anyhow::Result<()> {
         .open(&opt.database)?;
 
     // Open the LMDB database.
-    let index = Index::new(&env)?;
-
-    // Open the documents MTBL database.
-    let path = opt.database.join("documents.mtbl");
-    let file = File::open(path)?;
-    let mmap = unsafe { memmap::Mmap::map(&file)? };
-    let mmap = TransitiveArc(Arc::new(mmap));
-    let documents = Reader::new(mmap)?;
+    let index = Index::new(&env, &opt.database)?;
 
     // Retrieve the database the file stem (w/o the extension),
     // the disk file size and the number of documents in the database.
     let db_name = opt.database.file_stem().and_then(|s| s.to_str()).unwrap_or("").to_string();
     let db_size = File::open(opt.database.join("data.mdb"))?.metadata()?.len() as usize;
-
-    // Retrieve the documents count.
-    let docs_count = documents.metadata().count_entries;
+    let docs_count = index.number_of_documents();
 
     // We run and wait on the HTTP server
 
@@ -198,7 +188,6 @@ async fn main() -> anyhow::Result<()> {
     }
 
     let env_cloned = env.clone();
-    let documents_cloned = documents.clone();
     let disable_highlighting = opt.disable_highlighting;
     let query_route = warp::filters::method::post()
         .and(warp::path!("query"))
@@ -213,13 +202,10 @@ async fn main() -> anyhow::Result<()> {
             if let Some(headers) = index.headers(&rtxn).unwrap() {
                 // We write the headers
                 body.extend_from_slice(headers);
+                let documents = index.documents(documents_ids).unwrap();
 
-                for id in documents_ids {
-                    let id_bytes = id.to_be_bytes();
-                    let content = documents_cloned.clone().get(&id_bytes).unwrap();
-                    let content = content.expect(&format!("could not find document {}", id));
+                for (_id, content) in documents {
                     let content = std::str::from_utf8(content.as_ref()).unwrap();
-
                     let content = if disable_highlighting {
                         Cow::from(content)
                     } else {