Remove proximity database, forcing us to remove phrase search and splitwords

milli/src/documents/enriched.rs

@@ -1,5 +1,4 @@
 use std::fs::File;
-use std::io::BufReader;
 use std::{io, str};
 
 use obkv::KvReader;
@@ -20,14 +19,14 @@ use crate::FieldId;
 pub struct EnrichedDocumentsBatchReader<R> {
     documents: DocumentsBatchReader<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<BufReader<File>>,
+    external_ids: grenad::ReaderCursor<File>,
 }
 
 impl<R: io::Read + io::Seek> EnrichedDocumentsBatchReader<R> {
     pub fn new(
         documents: DocumentsBatchReader<R>,
         primary_key: String,
-        external_ids: grenad::Reader<BufReader<File>>,
+        external_ids: grenad::Reader<File>,
     ) -> Result<Self, Error> {
         if documents.documents_count() as u64 == external_ids.len() {
             Ok(EnrichedDocumentsBatchReader {
@@ -76,7 +75,7 @@ pub struct EnrichedDocument<'a> {
 pub struct EnrichedDocumentsBatchCursor<R> {
     documents: DocumentsBatchCursor<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<BufReader<File>>,
+    external_ids: grenad::ReaderCursor<File>,
 }
 
 impl<R> EnrichedDocumentsBatchCursor<R> {

milli/src/error.rs

@@ -47,8 +47,6 @@ pub enum InternalError {
     IndexingMergingKeys { process: &'static str },
     #[error("{}", HeedError::InvalidDatabaseTyping)]
     InvalidDatabaseTyping,
-    #[error("Could not access the inner of a buf-reader/writer: {0}")]
-    BufIntoInnerError(String),
     #[error(transparent)]
     RayonThreadPool(#[from] ThreadPoolBuildError),
     #[error(transparent)]

milli/src/search/new/db_cache.rs

@@ -12,7 +12,8 @@ use super::Word;
 use crate::heed_codec::{BytesDecodeOwned, StrBEU16Codec};
 use crate::update::{merge_cbo_roaring_bitmaps, MergeFn};
 use crate::{
-    CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec, SearchContext,
+    CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec,
+    RoaringBitmapLenCodec, SearchContext,
 };
 
 /// A cache storing pointers to values in the LMDB databases.
@@ -259,6 +260,7 @@ impl<'ctx> SearchContext<'ctx> {
         word2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
+        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (proximity, word1, word2),
@@ -278,6 +280,7 @@ impl<'ctx> SearchContext<'ctx> {
         word2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<u64>> {
+        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapLenCodec>(
             self.txn,
             (proximity, word1, word2),
@@ -291,12 +294,23 @@ impl<'ctx> SearchContext<'ctx> {
         )
     }
 
+    pub fn get_db_word_docids_len(&mut self, word: Interned<String>) -> Result<Option<u64>> {
+        DatabaseCache::get_value::<_, _, RoaringBitmapLenCodec>(
+            self.txn,
+            word,
+            self.word_interner.get(word).as_str(),
+            &mut self.db_cache.word_docids,
+            self.index.word_docids.remap_data_type::<ByteSlice>(),
+        )
+    }
+
     pub fn get_db_word_prefix_pair_proximity_docids(
         &mut self,
         word1: Interned<String>,
         prefix2: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
+        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (proximity, word1, prefix2),
@@ -315,6 +329,7 @@ impl<'ctx> SearchContext<'ctx> {
         right: Interned<String>,
         proximity: u8,
     ) -> Result<Option<RoaringBitmap>> {
+        unreachable!();
         DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             (proximity, left_prefix, right),

milli/src/search/new/mod.rs

@@ -295,11 +295,11 @@ fn get_ranking_rules_for_query_graph_search<'ctx>(
                 ranking_rules.push(Box::new(Typo::new(None)));
             }
             crate::Criterion::Proximity => {
-                if proximity {
-                    continue;
-                }
-                proximity = true;
-                ranking_rules.push(Box::new(Proximity::new(None)));
+                // if proximity {
+                continue;
+                // }
+                // proximity = true;
+                // ranking_rules.push(Box::new(Proximity::new(None)));
             }
             crate::Criterion::Attribute => {
                 if attribute {

milli/src/search/new/query_term/compute_derivations.rs

@@ -265,11 +265,11 @@ pub fn partially_initialized_term_from_word(
 }
 
 fn find_split_words(ctx: &mut SearchContext, word: &str) -> Result<Option<Interned<Phrase>>> {
-    if let Some((l, r)) = split_best_frequency(ctx, word)? {
-        Ok(Some(ctx.phrase_interner.insert(Phrase { words: vec![Some(l), Some(r)] })))
-    } else {
-        Ok(None)
-    }
+    // if let Some((l, r)) = split_best_frequency(ctx, word)? {
+    //     Ok(Some(ctx.phrase_interner.insert(Phrase { words: vec![Some(l), Some(r)] })))
+    // } else {
+    Ok(None)
+    // }
 }
 
 impl Interned<QueryTerm> {
@@ -416,11 +416,20 @@ fn split_best_frequency(
         let left = ctx.word_interner.insert(left.to_owned());
         let right = ctx.word_interner.insert(right.to_owned());
 
-        if let Some(frequency) = ctx.get_db_word_pair_proximity_docids_len(left, right, 1)? {
+        if let (Some(l_freq), Some(r_freq)) =
+            (ctx.get_db_word_docids_len(left)?, ctx.get_db_word_docids_len(right)?)
+        {
+            let frequency = l_freq.min(r_freq);
             if best.map_or(true, |(old, _, _)| frequency > old) {
                 best = Some((frequency, left, right));
             }
         }
+
+        // if let Some(frequency) = ctx.get_db_word_pair_proximity_docids_len(left, right, 1)? {
+        //     if best.map_or(true, |(old, _, _)| frequency > old) {
+        //         best = Some((frequency, left, right));
+        //     }
+        // }
     }
 
     Ok(best.map(|(_, left, right)| (left, right)))

milli/src/search/new/query_term/parse_query.rs

@@ -82,41 +82,41 @@ pub fn located_query_terms_from_tokens(
                     position = position.wrapping_add(7);
                 }
 
-                phrase = 'phrase: {
-                    let phrase = phrase.take();
+                // phrase = 'phrase: {
+                //     let phrase = phrase.take();
 
-                    // If we have a hard separator inside a phrase, we immediately start a new phrase
-                    let phrase = if separator_kind == SeparatorKind::Hard {
-                        if let Some(phrase) = phrase {
-                            if let Some(located_query_term) = phrase.build(ctx) {
-                                located_terms.push(located_query_term)
-                            }
-                            Some(PhraseBuilder::empty())
-                        } else {
-                            None
-                        }
-                    } else {
-                        phrase
-                    };
+                //     // If we have a hard separator inside a phrase, we immediately start a new phrase
+                //     let phrase = if separator_kind == SeparatorKind::Hard {
+                //         if let Some(phrase) = phrase {
+                //             if let Some(located_query_term) = phrase.build(ctx) {
+                //                 located_terms.push(located_query_term)
+                //             }
+                //             Some(PhraseBuilder::empty())
+                //         } else {
+                //             None
+                //         }
+                //     } else {
+                //         phrase
+                //     };
 
-                    // We close and start a new phrase depending on the number of double quotes
-                    let mut quote_count = token.lemma().chars().filter(|&s| s == '"').count();
-                    if quote_count == 0 {
-                        break 'phrase phrase;
-                    }
+                //     // We close and start a new phrase depending on the number of double quotes
+                //     let mut quote_count = token.lemma().chars().filter(|&s| s == '"').count();
+                //     if quote_count == 0 {
+                //         break 'phrase phrase;
+                //     }
 
-                    // Consume the closing quote and the phrase
-                    if let Some(phrase) = phrase {
-                        // Per the check above, quote_count > 0
-                        quote_count -= 1;
-                        if let Some(located_query_term) = phrase.build(ctx) {
-                            located_terms.push(located_query_term)
-                        }
-                    }
+                //     // Consume the closing quote and the phrase
+                //     if let Some(phrase) = phrase {
+                //         // Per the check above, quote_count > 0
+                //         quote_count -= 1;
+                //         if let Some(located_query_term) = phrase.build(ctx) {
+                //             located_terms.push(located_query_term)
+                //         }
+                //     }
 
-                    // Start new phrase if the token ends with an opening quote
-                    (quote_count % 2 == 1).then_some(PhraseBuilder::empty())
-                };
+                //     // Start new phrase if the token ends with an opening quote
+                //     (quote_count % 2 == 1).then_some(PhraseBuilder::empty())
+                // };
             }
             _ => (),
         }

milli/src/update/facet/bulk.rs

@@ -1,6 +1,5 @@
 use std::borrow::Cow;
 use std::fs::File;
-use std::io::BufReader;
 
 use grenad::CompressionType;
 use heed::types::ByteSlice;
@@ -31,7 +30,7 @@ pub struct FacetsUpdateBulk<'i> {
     facet_type: FacetType,
     field_ids: Vec<FieldId>,
     // None if level 0 does not need to be updated
-    new_data: Option<grenad::Reader<BufReader<File>>>,
+    new_data: Option<grenad::Reader<File>>,
 }
 
 impl<'i> FacetsUpdateBulk<'i> {
@@ -39,7 +38,7 @@ impl<'i> FacetsUpdateBulk<'i> {
         index: &'i Index,
         field_ids: Vec<FieldId>,
         facet_type: FacetType,
-        new_data: grenad::Reader<BufReader<File>>,
+        new_data: grenad::Reader<File>,
         group_size: u8,
         min_level_size: u8,
     ) -> FacetsUpdateBulk<'i> {
@@ -188,7 +187,7 @@ impl<R: std::io::Read + std::io::Seek> FacetsUpdateBulkInner<R> {
         &self,
         field_id: FieldId,
         txn: &RoTxn,
-    ) -> Result<(Vec<grenad::Reader<BufReader<File>>>, RoaringBitmap)> {
+    ) -> Result<(Vec<grenad::Reader<File>>, RoaringBitmap)> {
         let mut all_docids = RoaringBitmap::new();
         let subwriters = self.compute_higher_levels(txn, field_id, 32, &mut |bitmaps, _| {
             for bitmap in bitmaps {
@@ -260,7 +259,7 @@ impl<R: std::io::Read + std::io::Seek> FacetsUpdateBulkInner<R> {
         field_id: u16,
         level: u8,
         handle_group: &mut dyn FnMut(&[RoaringBitmap], &'t [u8]) -> Result<()>,
-    ) -> Result<Vec<grenad::Reader<BufReader<File>>>> {
+    ) -> Result<Vec<grenad::Reader<File>>> {
         if level == 0 {
             self.read_level_0(rtxn, field_id, handle_group)?;
             // Level 0 is already in the database

milli/src/update/facet/incremental.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::fs::File;
-use std::io::BufReader;
 
 use heed::types::{ByteSlice, DecodeIgnore};
 use heed::{BytesDecode, Error, RoTxn, RwTxn};
@@ -35,14 +34,14 @@ pub struct FacetsUpdateIncremental<'i> {
     index: &'i Index,
     inner: FacetsUpdateIncrementalInner,
     facet_type: FacetType,
-    new_data: grenad::Reader<BufReader<File>>,
+    new_data: grenad::Reader<File>,
 }
 
 impl<'i> FacetsUpdateIncremental<'i> {
     pub fn new(
         index: &'i Index,
         facet_type: FacetType,
-        new_data: grenad::Reader<BufReader<File>>,
+        new_data: grenad::Reader<File>,
         group_size: u8,
         min_level_size: u8,
         max_group_size: u8,

milli/src/update/facet/mod.rs

@@ -78,7 +78,6 @@ pub const FACET_MIN_LEVEL_SIZE: u8 = 5;
 
 use std::collections::BTreeSet;
 use std::fs::File;
-use std::io::BufReader;
 use std::iter::FromIterator;
 
 use charabia::normalizer::{Normalize, NormalizerOption};
@@ -109,17 +108,13 @@ pub struct FacetsUpdate<'i> {
     index: &'i Index,
     database: heed::Database<FacetGroupKeyCodec<ByteSliceRefCodec>, FacetGroupValueCodec>,
     facet_type: FacetType,
-    new_data: grenad::Reader<BufReader<File>>,
+    new_data: grenad::Reader<File>,
     group_size: u8,
     max_group_size: u8,
     min_level_size: u8,
 }
 impl<'i> FacetsUpdate<'i> {
-    pub fn new(
-        index: &'i Index,
-        facet_type: FacetType,
-        new_data: grenad::Reader<BufReader<File>>,
-    ) -> Self {
+    pub fn new(index: &'i Index, facet_type: FacetType, new_data: grenad::Reader<File>) -> Self {
         let database = match facet_type {
             FacetType::String => index
                 .facet_id_string_docids

milli/src/update/index_documents/enrich.rs

@@ -1,4 +1,4 @@
-use std::io::{BufWriter, Read, Seek};
+use std::io::{Read, Seek};
 use std::result::Result as StdResult;
 use std::{fmt, iter};
 
@@ -35,7 +35,7 @@ pub fn enrich_documents_batch<R: Read + Seek>(
 
     let (mut cursor, mut documents_batch_index) = reader.into_cursor_and_fields_index();
 
-    let mut external_ids = tempfile::tempfile().map(BufWriter::new).map(grenad::Writer::new)?;
+    let mut external_ids = tempfile::tempfile().map(grenad::Writer::new)?;
     let mut uuid_buffer = [0; uuid::fmt::Hyphenated::LENGTH];
 
     // The primary key *field id* that has already been set for this index or the one

milli/src/update/index_documents/extract/extract_docid_word_positions.rs

@@ -1,7 +1,6 @@
 use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
-use std::io::BufReader;
 use std::{io, mem, str};
 
 use charabia::{Language, Script, SeparatorKind, Token, TokenKind, Tokenizer, TokenizerBuilder};
@@ -32,7 +31,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
     allowed_separators: Option<&[&str]>,
     dictionary: Option<&[&str]>,
     max_positions_per_attributes: Option<u32>,
-) -> Result<(RoaringBitmap, grenad::Reader<BufReader<File>>, ScriptLanguageDocidsMap)> {
+) -> Result<(RoaringBitmap, grenad::Reader<File>, ScriptLanguageDocidsMap)> {
     puffin::profile_function!();
 
     let max_positions_per_attributes = max_positions_per_attributes

milli/src/update/index_documents/extract/extract_facet_number_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use heed::{BytesDecode, BytesEncode};
 
@@ -19,7 +19,7 @@ use crate::Result;
 pub fn extract_facet_number_docids<R: io::Read + io::Seek>(
     docid_fid_facet_number: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_facet_string_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use heed::BytesEncode;
 
@@ -17,7 +17,7 @@ use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
 pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
     docid_fid_facet_string: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_fid_docid_facet_values.rs

@@ -1,7 +1,7 @@
 use std::collections::{BTreeMap, HashSet};
 use std::convert::TryInto;
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 use std::mem::size_of;
 
 use heed::zerocopy::AsBytes;
@@ -17,11 +17,11 @@ use crate::{CboRoaringBitmapCodec, DocumentId, FieldId, Result, BEU32, MAX_FACET
 
 /// The extracted facet values stored in grenad files by type.
 pub struct ExtractedFacetValues {
-    pub docid_fid_facet_numbers_chunk: grenad::Reader<BufReader<File>>,
-    pub docid_fid_facet_strings_chunk: grenad::Reader<BufReader<File>>,
-    pub fid_facet_is_null_docids_chunk: grenad::Reader<BufReader<File>>,
-    pub fid_facet_is_empty_docids_chunk: grenad::Reader<BufReader<File>>,
-    pub fid_facet_exists_docids_chunk: grenad::Reader<BufReader<File>>,
+    pub docid_fid_facet_numbers_chunk: grenad::Reader<File>,
+    pub docid_fid_facet_strings_chunk: grenad::Reader<File>,
+    pub fid_facet_is_null_docids_chunk: grenad::Reader<File>,
+    pub fid_facet_is_empty_docids_chunk: grenad::Reader<File>,
+    pub fid_facet_exists_docids_chunk: grenad::Reader<File>,
 }
 
 /// Extracts the facet values of each faceted field of each document.

milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use grenad::Sorter;
 
@@ -21,7 +21,7 @@ use crate::{relative_from_absolute_position, DocumentId, FieldId, Result};
 pub fn extract_fid_word_count_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_geo_points.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use concat_arrays::concat_arrays;
 use serde_json::Value;
@@ -18,7 +18,7 @@ pub fn extract_geo_points<R: io::Read + io::Seek>(
     indexer: GrenadParameters,
     primary_key_id: FieldId,
     (lat_fid, lng_fid): (FieldId, FieldId),
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let mut writer = create_writer(

milli/src/update/index_documents/extract/extract_vector_points.rs

@@ -1,6 +1,6 @@
 use std::convert::TryFrom;
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use bytemuck::cast_slice;
 use serde_json::{from_slice, Value};
@@ -18,7 +18,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
     indexer: GrenadParameters,
     primary_key_id: FieldId,
     vectors_fid: FieldId,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let mut writer = create_writer(

milli/src/update/index_documents/extract/extract_word_docids.rs

@@ -1,6 +1,6 @@
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 use std::iter::FromIterator;
 
 use roaring::RoaringBitmap;
@@ -26,7 +26,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
     exact_attributes: &HashSet<FieldId>,
-) -> Result<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
+) -> Result<(grenad::Reader<File>, grenad::Reader<File>)> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_fid_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use super::helpers::{
     create_sorter, merge_cbo_roaring_bitmaps, read_u32_ne_bytes, sorter_into_reader,
@@ -14,7 +14,7 @@ use crate::{relative_from_absolute_position, DocumentId, Result};
 pub fn extract_word_fid_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs

@@ -1,7 +1,6 @@
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
 use std::fs::File;
-use std::io::BufReader;
 use std::{cmp, io, mem, str, vec};
 
 use super::helpers::{
@@ -21,7 +20,7 @@ use crate::{DocumentId, Result};
 pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/extract_word_position_docids.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use super::helpers::{
     create_sorter, merge_cbo_roaring_bitmaps, read_u32_ne_bytes, sorter_into_reader,
@@ -17,7 +17,7 @@ use crate::{bucketed_position, relative_from_absolute_position, DocumentId, Resu
 pub fn extract_word_position_docids<R: io::Read + io::Seek>(
     docid_word_positions: grenad::Reader<R>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     puffin::profile_function!();
 
     let max_memory = indexer.max_memory_by_thread();

milli/src/update/index_documents/extract/mod.rs

@@ -12,7 +12,6 @@ mod extract_word_position_docids;
 
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::BufReader;
 
 use crossbeam_channel::Sender;
 use log::debug;
@@ -40,8 +39,8 @@ use crate::{FieldId, Result};
 /// Send data in grenad file over provided Sender.
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn data_from_obkv_documents(
-    original_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
-    flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>> + Send,
+    original_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<File>>> + Send,
+    flattened_obkv_chunks: impl Iterator<Item = Result<grenad::Reader<File>>> + Send,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     searchable_fields: Option<HashSet<FieldId>>,
@@ -153,17 +152,17 @@ pub(crate) fn data_from_obkv_documents(
         });
     }
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
-        docid_word_positions_chunks.clone(),
-        indexer,
-        lmdb_writer_sx.clone(),
-        extract_word_pair_proximity_docids,
-        merge_cbo_roaring_bitmaps,
-        TypedChunk::WordPairProximityDocids,
-        "word-pair-proximity-docids",
-    );
+    // spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
+    //     docid_word_positions_chunks.clone(),
+    //     indexer,
+    //     lmdb_writer_sx.clone(),
+    //     extract_word_pair_proximity_docids,
+    //     merge_cbo_roaring_bitmaps,
+    //     TypedChunk::WordPairProximityDocids,
+    //     "word-pair-proximity-docids",
+    // );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -173,11 +172,7 @@ pub(crate) fn data_from_obkv_documents(
         "field-id-wordcount-docids",
     );
 
-    spawn_extraction_task::<
-        _,
-        _,
-        Vec<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)>,
-    >(
+    spawn_extraction_task::<_, _, Vec<(grenad::Reader<File>, grenad::Reader<File>)>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -190,7 +185,7 @@ pub(crate) fn data_from_obkv_documents(
         "word-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
         docid_word_positions_chunks.clone(),
         indexer,
         lmdb_writer_sx.clone(),
@@ -199,7 +194,7 @@ pub(crate) fn data_from_obkv_documents(
         TypedChunk::WordPositionDocids,
         "word-position-docids",
     );
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
         docid_word_positions_chunks,
         indexer,
         lmdb_writer_sx.clone(),
@@ -209,7 +204,7 @@ pub(crate) fn data_from_obkv_documents(
         "word-fid-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
         docid_fid_facet_strings_chunks,
         indexer,
         lmdb_writer_sx.clone(),
@@ -219,7 +214,7 @@ pub(crate) fn data_from_obkv_documents(
         "field-id-facet-string-docids",
     );
 
-    spawn_extraction_task::<_, _, Vec<grenad::Reader<BufReader<File>>>>(
+    spawn_extraction_task::<_, _, Vec<grenad::Reader<File>>>(
         docid_fid_facet_numbers_chunks,
         indexer,
         lmdb_writer_sx,
@@ -274,7 +269,7 @@ fn spawn_extraction_task<FE, FS, M>(
 /// Extract chunked data and send it into lmdb_writer_sx sender:
 /// - documents
 fn send_original_documents_data(
-    original_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
+    original_documents_chunk: Result<grenad::Reader<File>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     vectors_field_id: Option<FieldId>,
@@ -316,7 +311,7 @@ fn send_original_documents_data(
 #[allow(clippy::too_many_arguments)]
 #[allow(clippy::type_complexity)]
 fn send_and_extract_flattened_documents_data(
-    flattened_documents_chunk: Result<grenad::Reader<BufReader<File>>>,
+    flattened_documents_chunk: Result<grenad::Reader<File>>,
     indexer: GrenadParameters,
     lmdb_writer_sx: Sender<Result<TypedChunk>>,
     searchable_fields: &Option<HashSet<FieldId>>,
@@ -333,10 +328,7 @@ fn send_and_extract_flattened_documents_data(
         grenad::Reader<CursorClonableMmap>,
         (
             grenad::Reader<CursorClonableMmap>,
-            (
-                grenad::Reader<BufReader<File>>,
-                (grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>),
-            ),
+            (grenad::Reader<File>, (grenad::Reader<File>, grenad::Reader<File>)),
         ),
     ),
 )> {

milli/src/update/index_documents/helpers/grenad_helpers.rs

@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 use std::fs::File;
-use std::io::{self, BufReader, BufWriter, Seek};
+use std::io::{self, Seek};
 use std::time::Instant;
 
 use grenad::{CompressionType, Sorter};
@@ -17,13 +17,13 @@ pub fn create_writer<R: io::Write>(
     typ: grenad::CompressionType,
     level: Option<u32>,
     file: R,
-) -> grenad::Writer<BufWriter<R>> {
+) -> grenad::Writer<R> {
     let mut builder = grenad::Writer::builder();
     builder.compression_type(typ);
     if let Some(level) = level {
         builder.compression_level(level);
     }
-    builder.build(BufWriter::new(file))
+    builder.build(file)
 }
 
 pub fn create_sorter(
@@ -53,7 +53,7 @@ pub fn create_sorter(
 pub fn sorter_into_reader(
     sorter: grenad::Sorter<MergeFn>,
     indexer: GrenadParameters,
-) -> Result<grenad::Reader<BufReader<File>>> {
+) -> Result<grenad::Reader<File>> {
     let mut writer = create_writer(
         indexer.chunk_compression_type,
         indexer.chunk_compression_level,
@@ -64,21 +64,16 @@ pub fn sorter_into_reader(
     writer_into_reader(writer)
 }
 
-pub fn writer_into_reader(
-    writer: grenad::Writer<BufWriter<File>>,
-) -> Result<grenad::Reader<BufReader<File>>> {
-    let mut file = writer
-        .into_inner()?
-        .into_inner()
-        .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?;
+pub fn writer_into_reader(writer: grenad::Writer<File>) -> Result<grenad::Reader<File>> {
+    let mut file = writer.into_inner()?;
     file.rewind()?;
-    grenad::Reader::new(BufReader::new(file)).map_err(Into::into)
+    grenad::Reader::new(file).map_err(Into::into)
 }
 
 pub unsafe fn as_cloneable_grenad(
-    reader: &grenad::Reader<BufReader<File>>,
+    reader: &grenad::Reader<File>,
 ) -> Result<grenad::Reader<CursorClonableMmap>> {
-    let file = reader.get_ref().get_ref();
+    let file = reader.get_ref();
     let mmap = memmap2::Mmap::map(file)?;
     let cursor = io::Cursor::new(ClonableMmap::from(mmap));
     let reader = grenad::Reader::new(cursor)?;
@@ -94,8 +89,8 @@ where
     fn merge(self, merge_fn: MergeFn, indexer: &GrenadParameters) -> Result<Self::Output>;
 }
 
-impl MergeableReader for Vec<grenad::Reader<BufReader<File>>> {
-    type Output = grenad::Reader<BufReader<File>>;
+impl MergeableReader for Vec<grenad::Reader<File>> {
+    type Output = grenad::Reader<File>;
 
     fn merge(self, merge_fn: MergeFn, params: &GrenadParameters) -> Result<Self::Output> {
         let mut merger = MergerBuilder::new(merge_fn);
@@ -104,8 +99,8 @@ impl MergeableReader for Vec<grenad::Reader<BufReader<File>>> {
     }
 }
 
-impl MergeableReader for Vec<(grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>)> {
-    type Output = (grenad::Reader<BufReader<File>>, grenad::Reader<BufReader<File>>);
+impl MergeableReader for Vec<(grenad::Reader<File>, grenad::Reader<File>)> {
+    type Output = (grenad::Reader<File>, grenad::Reader<File>);
 
     fn merge(self, merge_fn: MergeFn, params: &GrenadParameters) -> Result<Self::Output> {
         let mut m1 = MergerBuilder::new(merge_fn);
@@ -130,7 +125,7 @@ impl<R: io::Read + io::Seek> MergerBuilder<R> {
         Ok(())
     }
 
-    fn finish(self, params: &GrenadParameters) -> Result<grenad::Reader<BufReader<File>>> {
+    fn finish(self, params: &GrenadParameters) -> Result<grenad::Reader<File>> {
         let merger = self.0.build();
         let mut writer = create_writer(
             params.chunk_compression_type,
@@ -181,7 +176,7 @@ pub fn grenad_obkv_into_chunks<R: io::Read + io::Seek>(
     reader: grenad::Reader<R>,
     indexer: GrenadParameters,
     documents_chunk_size: usize,
-) -> Result<impl Iterator<Item = Result<grenad::Reader<BufReader<File>>>>> {
+) -> Result<impl Iterator<Item = Result<grenad::Reader<File>>>> {
     let mut continue_reading = true;
     let mut cursor = reader.into_cursor()?;
 

milli/src/update/index_documents/transform.rs

@@ -659,12 +659,8 @@ impl<'a, 'i> Transform<'a, 'i> {
             new_documents_ids: self.new_documents_ids,
             replaced_documents_ids: self.replaced_documents_ids,
             documents_count: self.documents_count,
-            original_documents: original_documents
-                .into_inner()
-                .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?,
-            flattened_documents: flattened_documents
-                .into_inner()
-                .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?,
+            original_documents,
+            flattened_documents,
         })
     }
 
@@ -783,12 +779,8 @@ impl<'a, 'i> Transform<'a, 'i> {
             new_documents_ids: documents_ids,
             replaced_documents_ids: RoaringBitmap::default(),
             documents_count,
-            original_documents: original_documents
-                .into_inner()
-                .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?,
-            flattened_documents: flattened_documents
-                .into_inner()
-                .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?,
+            original_documents,
+            flattened_documents,
         };
 
         let new_facets = output.compute_real_facets(wtxn, self.index)?;

milli/src/update/index_documents/typed_chunk.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fs::File;
-use std::io::{self, BufReader};
+use std::io;
 
 use bytemuck::allocation::pod_collect_to_vec;
 use charabia::{Language, Script};
@@ -27,22 +27,22 @@ pub(crate) enum TypedChunk {
     FieldIdDocidFacetStrings(grenad::Reader<CursorClonableMmap>),
     FieldIdDocidFacetNumbers(grenad::Reader<CursorClonableMmap>),
     Documents(grenad::Reader<CursorClonableMmap>),
-    FieldIdWordcountDocids(grenad::Reader<BufReader<File>>),
+    FieldIdWordcountDocids(grenad::Reader<File>),
     NewDocumentsIds(RoaringBitmap),
     WordDocids {
-        word_docids_reader: grenad::Reader<BufReader<File>>,
-        exact_word_docids_reader: grenad::Reader<BufReader<File>>,
+        word_docids_reader: grenad::Reader<File>,
+        exact_word_docids_reader: grenad::Reader<File>,
     },
-    WordPositionDocids(grenad::Reader<BufReader<File>>),
-    WordFidDocids(grenad::Reader<BufReader<File>>),
-    WordPairProximityDocids(grenad::Reader<BufReader<File>>),
-    FieldIdFacetStringDocids(grenad::Reader<BufReader<File>>),
-    FieldIdFacetNumberDocids(grenad::Reader<BufReader<File>>),
-    FieldIdFacetExistsDocids(grenad::Reader<BufReader<File>>),
-    FieldIdFacetIsNullDocids(grenad::Reader<BufReader<File>>),
-    FieldIdFacetIsEmptyDocids(grenad::Reader<BufReader<File>>),
-    GeoPoints(grenad::Reader<BufReader<File>>),
-    VectorPoints(grenad::Reader<BufReader<File>>),
+    WordPositionDocids(grenad::Reader<File>),
+    WordFidDocids(grenad::Reader<File>),
+    WordPairProximityDocids(grenad::Reader<File>),
+    FieldIdFacetStringDocids(grenad::Reader<File>),
+    FieldIdFacetNumberDocids(grenad::Reader<File>),
+    FieldIdFacetExistsDocids(grenad::Reader<File>),
+    FieldIdFacetIsNullDocids(grenad::Reader<File>),
+    FieldIdFacetIsEmptyDocids(grenad::Reader<File>),
+    GeoPoints(grenad::Reader<File>),
+    VectorPoints(grenad::Reader<File>),
     ScriptLanguageDocids(HashMap<(Script, Language), RoaringBitmap>),
 }
 

milli/src/update/prefix_word_pairs/mod.rs

@@ -1,12 +1,12 @@
 use std::borrow::Cow;
 use std::collections::HashSet;
-use std::io::{BufReader, BufWriter};
+use std::io::BufReader;
 
 use grenad::CompressionType;
 use heed::types::ByteSlice;
 
 use super::index_documents::{merge_cbo_roaring_bitmaps, CursorClonableMmap};
-use crate::{Index, InternalError, Result};
+use crate::{Index, Result};
 
 mod prefix_word;
 mod word_prefix;
@@ -119,12 +119,9 @@ pub fn insert_into_database(
 pub fn write_into_lmdb_database_without_merging(
     wtxn: &mut heed::RwTxn,
     database: heed::PolyDatabase,
-    writer: grenad::Writer<BufWriter<std::fs::File>>,
+    writer: grenad::Writer<std::fs::File>,
 ) -> Result<()> {
-    let file = writer
-        .into_inner()?
-        .into_inner()
-        .map_err(|err| InternalError::BufIntoInnerError(err.to_string()))?;
+    let file = writer.into_inner()?;
     let reader = grenad::Reader::new(BufReader::new(file))?;
     if database.is_empty(wtxn)? {
         let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;