snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-08-02 11:50:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix the computation of the newly added and common prefix words

Browse Source
This commit is contained in:
Clément Renault
2022-01-20 17:55:52 +01:00
committed by Kerollmops
parent 2ec8542105
commit d59e559317
3 changed files with 32 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
milli/src/update/index_documents/helpers/mod.rs
View File

@ -46,7 +46,7 @@ pub fn read_u32_ne_bytes(bytes: &[u8]) -> impl Iterator<Item = u32> + '_ {
bytes.chunks_exact(4).flat_map(TryInto::try_into).map(u32::from_ne_bytes)
}
/// Converts an fst Stream into an HashSet.
/// Converts an fst Stream into an HashSet of Strings.
pub fn fst_stream_into_hashset<'f, I, S>(stream: I) -> HashSet<Vec<u8>>
where
I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
@ -59,3 +59,18 @@ where
}
hashset
}
// Converts an fst Stream into a Vec of Strings.
pub fn fst_stream_into_vec<'f, I, S>(stream: I) -> Vec<String>
where
I: for<'a> IntoStreamer<'a, Into = S, Item = &'a [u8]>,
S: 'f + for<'a> Streamer<'a, Item = &'a [u8]>,
{
let mut strings = Vec::new();
let mut stream = stream.into_stream();
while let Some(word) = stream.next() {
let s = std::str::from_utf8(word).unwrap();
strings.push(s.to_owned());
}
strings
}

6
milli/src/update/index_documents/mod.rs
View File

@ -15,9 +15,9 @@ use serde::{Deserialize, Serialize};
use typed_chunk::{write_typed_chunk_into_index, TypedChunk};
pub use self::helpers::{
create_sorter, create_writer, fst_stream_into_hashset, merge_cbo_roaring_bitmaps,
merge_roaring_bitmaps, sorter_into_lmdb_database, write_into_lmdb_database, writer_into_reader,
ClonableMmap, MergeFn,
create_sorter, create_writer, fst_stream_into_hashset, fst_stream_into_vec,
merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, sorter_into_lmdb_database,
write_into_lmdb_database, writer_into_reader, ClonableMmap, MergeFn,
};
use self::helpers::{grenad_obkv_into_chunks, GrenadParameters};
pub use self::transform::{Transform, TransformOutput};