snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-06-06 04:05:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Log when an entry is too large to fit in the BBQueue

Browse Source
This commit is contained in:
Kerollmops 2024-12-05 11:24:16 +01:00
parent cac355bfa7
commit 63b6229984
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crates/milli/src/update/new/channel.rs
View File

@ -480,6 +480,8 @@ impl<'b> ExtractorBbqueueSender<'b> {
let payload_header = EntryHeader::ArroySetVectors(arroy_set_vector);
let total_length = EntryHeader::total_set_vectors_size(embeddings.len(), dimensions);
if total_length > capacity {
tracing::trace!("We are spilling a large vector that is {total_length} bytes which is larger than the capacity of {capacity} bytes");
let mut value_file = tempfile::tempfile().map(BufWriter::new)?;
for embedding in embeddings {
let mut embedding_bytes = bytemuck::cast_slice(embedding);
@ -548,6 +550,8 @@ impl<'b> ExtractorBbqueueSender<'b> {
let payload_header = EntryHeader::DbOperation(operation);
let total_length = EntryHeader::total_key_value_size(key_length, value_length);
if total_length > capacity {
tracing::trace!("We are spilling a large entry that is {total_length} bytes which is larger than the capacity of {capacity} bytes");
let mut key_buffer = vec![0; key_length.get() as usize].into_boxed_slice();
let value_file = tempfile::tempfile()?;
value_file.set_len(value_length.try_into().unwrap())?;