Bump heed to use the git repo with v0.12.0

milli/Cargo.toml

@@ -14,7 +14,7 @@ flate2 = "1.0.20"
 fst = "0.4.5"
 fxhash = "0.2.1"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] }
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] }
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"

milli/src/update/delete_documents.rs

@@ -132,7 +132,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     };
                     external_ids.push(external_id);
                 }
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
             drop(iter);
 
@@ -143,7 +144,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                 let ((_docid, word), _positions) = result?;
                 // This boolean will indicate if we must remove this word from the words FST.
                 words.push((SmallString32::from(word), false));
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
         }
 
@@ -194,11 +196,13 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     let previous_len = docids.len();
                     docids.difference_with(&self.documents_ids);
                     if docids.is_empty() {
-                        iter.del_current()?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.del_current()? };
                         *must_remove = true;
                     } else if docids.len() != previous_len {
                         let key = key.to_owned();
-                        iter.put_current(&key, &docids)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(&key, &docids)? };
                     }
                 }
             }
@@ -243,10 +247,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
                 prefixes_to_delete.insert(prefix)?;
             } else if docids.len() != previous_len {
-                iter.put_current(&prefix, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&prefix, &docids)? };
             }
         }
 
@@ -281,10 +287,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let key = key.to_owned();
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -300,10 +308,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -317,10 +327,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -334,10 +346,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -349,9 +363,11 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -420,7 +436,8 @@ where
     while let Some(result) = iter.next() {
         let (key, ()) = result?;
         if to_remove.contains(convert(key)) {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         }
     }
 
@@ -441,10 +458,12 @@ where
         let previous_len = docids.len();
         docids.difference_with(to_remove);
         if docids.is_empty() {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         } else if docids.len() != previous_len {
             let bytes = bytes.to_owned();
-            iter.put_current(&bytes, &docids)?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.put_current(&bytes, &docids)? };
         }
     }
 

milli/src/update/index_documents/mod.rs

@@ -144,7 +144,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = reader.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -154,7 +155,8 @@ where
                     Some((key, old_val)) if key == k => {
                         let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
                         let val = merge(k, &vals)?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);
@@ -203,7 +205,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = sorter.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -216,7 +219,8 @@ where
                             // TODO just wrap this error?
                             InternalError::IndexingMergingKeys { process: "get-put-merge" }
                         })?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);