Merge #269 #271

269: Fix bug when inserting previously deleted documents r=Kerollmops a=Kerollmops

This PR fixes #268.

The issue was in the `ExternalDocumentsIds` implementation in the specific case that an external document id was in the soft map marked as deleted.

The bug was due to a wrong assumption on my side about how the FST unions were returning the `IndexedValue`s, I thought the values returned in an array were in the same order as the FSTs given to the `OpBuilder` but in fact, [the `IndexedValue`'s `index` field was here to indicate from which FST the values were coming from](https://docs.rs/fst/0.4.7/fst/map/struct.IndexedValue.html).

271: Remove the roaring operation functions warnings r=Kerollmops a=Kerollmops

In this PR we are just replacing the usages of the roaring operations function by the new operators. This removes a lot of warnings.

Co-authored-by: Kerollmops <clement@meilisearch.com>

milli/src/update/index_documents/merge_function.rs

@@ -61,8 +61,7 @@ pub fn roaring_bitmap_merge(_key: &[u8], values: &[Cow<[u8]>]) -> Result<Vec<u8>
     let mut head = RoaringBitmap::deserialize_from(&head[..])?;
 
     for value in tail {
-        let bitmap = RoaringBitmap::deserialize_from(&value[..])?;
-        head.union_with(&bitmap);
+        head |= RoaringBitmap::deserialize_from(&value[..])?;
     }
 
     let mut vec = Vec::with_capacity(head.serialized_size());
@@ -75,8 +74,7 @@ pub fn cbo_roaring_bitmap_merge(_key: &[u8], values: &[Cow<[u8]>]) -> Result<Vec
     let mut head = CboRoaringBitmapCodec::deserialize_from(&head[..])?;
 
     for value in tail {
-        let bitmap = CboRoaringBitmapCodec::deserialize_from(&value[..])?;
-        head.union_with(&bitmap);
+        head |= CboRoaringBitmapCodec::deserialize_from(&value[..])?;
     }
 
     let mut vec = Vec::new();

milli/src/update/index_documents/mod.rs

@@ -608,8 +608,8 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
         self.index.put_external_documents_ids(self.wtxn, &external_documents_ids)?;
 
         // We merge the new documents ids with the existing ones.
-        documents_ids.union_with(&new_documents_ids);
-        documents_ids.union_with(&replaced_documents_ids);
+        documents_ids |= new_documents_ids;
+        documents_ids |= replaced_documents_ids;
         self.index.put_documents_ids(self.wtxn, &documents_ids)?;
 
         let mut database_count = 0;
@@ -845,6 +845,7 @@ mod tests {
     use heed::EnvOpenOptions;
 
     use super::*;
+    use crate::update::DeleteDocuments;
 
     #[test]
     fn simple_document_replacement() {
@@ -1303,4 +1304,52 @@ mod tests {
         builder.execute(Cursor::new(documents), |_, _| ()).unwrap();
         wtxn.commit().unwrap();
     }
+
+    #[test]
+    fn delete_documents_then_insert() {
+        let path = tempfile::tempdir().unwrap();
+        let mut options = EnvOpenOptions::new();
+        options.map_size(10 * 1024 * 1024); // 10 MB
+        let index = Index::new(options, &path).unwrap();
+
+        let mut wtxn = index.write_txn().unwrap();
+        let content = &br#"[
+            { "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" },
+            { "objectId": 456, "title": "Le Petit Prince",     "comment": "A french book" },
+            { "objectId": 1,   "title": "Alice In Wonderland", "comment": "A weird book" },
+            { "objectId": 30,  "title": "Hamlet" }
+        ]"#[..];
+        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
+        builder.update_format(UpdateFormat::Json);
+        builder.execute(content, |_, _| ()).unwrap();
+
+        assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId"));
+
+        // Delete not all of the documents but some of them.
+        let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
+        builder.delete_external_id("30");
+        builder.execute().unwrap();
+
+        let external_documents_ids = index.external_documents_ids(&wtxn).unwrap();
+        assert!(external_documents_ids.get("30").is_none());
+
+        let content = &br#"[
+            { "objectId": 30, "title": "Hamlet" }
+        ]"#[..];
+        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
+        builder.update_format(UpdateFormat::Json);
+        builder.execute(content, |_, _| ()).unwrap();
+
+        let external_documents_ids = index.external_documents_ids(&wtxn).unwrap();
+        assert!(external_documents_ids.get("30").is_some());
+
+        let content = &br#"[
+            { "objectId": 30, "title": "Hamlet" }
+        ]"#[..];
+        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
+        builder.update_format(UpdateFormat::Json);
+        builder.execute(content, |_, _| ()).unwrap();
+
+        wtxn.commit().unwrap();
+    }
 }