Support diff indexing on extract_docid_word_positions

This commit is contained in:
ManyTheFish
2023-10-16 14:58:11 +02:00
committed by Louis Dureuil
parent 1dd97578a8
commit 313b16bec2
5 changed files with 322 additions and 131 deletions

View File

@ -805,7 +805,7 @@ impl<'a, 'i> Transform<'a, 'i> {
let buffer = obkv_writer.into_inner()?;
document_sorter_buffer.clear();
into_del_add_obkv(KvReaderU16::new(buffer), true, true, &mut document_sorter_buffer)?;
into_del_add_obkv(KvReaderU16::new(buffer), false, true, &mut document_sorter_buffer)?;
original_writer.insert(docid.to_be_bytes(), &document_sorter_buffer)?;
// Once we have the document. We're going to flatten it
@ -842,7 +842,7 @@ impl<'a, 'i> Transform<'a, 'i> {
writer.insert(fid, &value)?;
}
document_sorter_buffer.clear();
into_del_add_obkv(KvReaderU16::new(&buffer), true, true, &mut document_sorter_buffer)?;
into_del_add_obkv(KvReaderU16::new(&buffer), false, true, &mut document_sorter_buffer)?;
flattened_writer.insert(docid.to_be_bytes(), &document_sorter_buffer)?;
}