snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-31 07:56:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #5413

Browse Source 
5413: Make sure to delete useless prefixes r=ManyTheFish a=Kerollmops

We discovered a bug where the new indexer was still writing empty roaring bitmaps instead of deleting the prefix entry from the prefix database.

Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
meili-bors[bot]
2025-03-12 10:54:04 +00:00
committed by GitHub
parent a2a86ef4e2 0197dc87e0
commit 1cd00f37c0
1 changed files with 31 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
crates/milli/src/update/new/words_prefix_docids.rs
View File

@@ -205,15 +205,22 @@ impl WordPrefixIntegerDocids {
let (ref mut index, ref mut file, ref mut buffer) = *refmut; let (ref mut index, ref mut file, ref mut buffer) = *refmut;
for (&pos, bitmaps_bytes) in frozen.bitmaps(prefix).unwrap() { for (&pos, bitmaps_bytes) in frozen.bitmaps(prefix).unwrap() {
let output = bitmaps_bytes if bitmaps_bytes.is_empty() {
.iter() index.push(PrefixIntegerEntry { prefix, pos, serialized_length: None });
.map(|bytes| CboRoaringBitmapCodec::deserialize_from(bytes)) } else {
.union()?; let output = bitmaps_bytes
.iter()
buffer.clear(); .map(|bytes| CboRoaringBitmapCodec::deserialize_from(bytes))
CboRoaringBitmapCodec::serialize_into_vec(&output, buffer); .union()?;
index.push(PrefixIntegerEntry { prefix, pos, serialized_length: buffer.len() }); buffer.clear();
file.write_all(buffer)?; CboRoaringBitmapCodec::serialize_into_vec(&output, buffer);
index.push(PrefixIntegerEntry {
prefix,
pos,
serialized_length: Some(buffer.len()),
});
file.write_all(buffer)?;
}
} }
Result::Ok(()) Result::Ok(())
@@ -230,14 +237,24 @@ impl WordPrefixIntegerDocids {
file.rewind()?; file.rewind()?;
let mut file = BufReader::new(file); let mut file = BufReader::new(file);
for PrefixIntegerEntry { prefix, pos, serialized_length } in index { for PrefixIntegerEntry { prefix, pos, serialized_length } in index {
buffer.resize(serialized_length, 0);
file.read_exact(&mut buffer)?;
key_buffer.clear(); key_buffer.clear();
key_buffer.extend_from_slice(prefix.as_bytes()); key_buffer.extend_from_slice(prefix.as_bytes());
key_buffer.push(0); key_buffer.push(0);
key_buffer.extend_from_slice(&pos.to_be_bytes()); key_buffer.extend_from_slice(&pos.to_be_bytes());
self.prefix_database.remap_data_type::<Bytes>().put(wtxn, &key_buffer, &buffer)?; match serialized_length {
Some(serialized_length) => {
buffer.resize(serialized_length, 0);
file.read_exact(&mut buffer)?;
self.prefix_database.remap_data_type::<Bytes>().put(
wtxn,
&key_buffer,
&buffer,
)?;
}
None => {
self.prefix_database.delete(wtxn, &key_buffer)?;
}
}
} }
} }
@@ -249,7 +266,7 @@ impl WordPrefixIntegerDocids {
struct PrefixIntegerEntry<'a> { struct PrefixIntegerEntry<'a> {
prefix: &'a str, prefix: &'a str,
pos: u16, pos: u16,
serialized_length: usize, serialized_length: Option<usize>,
} }
/// TODO doc /// TODO doc