snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-11 22:26:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fasten the document deletion

Browse Source 
When a document deletion occurs, instead of deleting the document we mark it as deleted
in the new “soft deleted” bitmap. It is then removed from the search, and all the other
endpoints.
This commit is contained in:
Tamo
2022-06-13 17:59:34 +02:00
parent 77c837fc1b
commit 3b309f654a
10 changed files with 413 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

376
milli/src/update/delete_documents.rs
View File

@@ -1,5 +1,4 @@
use std::collections::btree_map::Entry;
use std::collections::HashMap;
use fst::IntoStreamer;
use heed::types::{ByteSlice, Str};
@@ -17,15 +16,17 @@ use crate::heed_codec::facet::{
use crate::heed_codec::CboRoaringBitmapCodec;
use crate::index::{db_name, main_key};
use crate::{
DocumentId, ExternalDocumentsIds, FieldId, Index, Result, RoaringBitmapCodec, SmallString32,
BEU32,
DocumentId, ExternalDocumentsIds, FieldId, FieldIdMapMissingEntry, Index, Result,
RoaringBitmapCodec, SmallString32, BEU32,
};
const DELETE_DOCUMENTS_THRESHOLD: u64 = 100_000;
pub struct DeleteDocuments<'t, 'u, 'i> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
external_documents_ids: ExternalDocumentsIds<'static>,
documents_ids: RoaringBitmap,
to_delete_docids: RoaringBitmap,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -45,16 +46,16 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
wtxn,
index,
external_documents_ids,
documents_ids: RoaringBitmap::new(),
to_delete_docids: RoaringBitmap::new(),
})
}
pub fn delete_document(&mut self, docid: u32) {
self.documents_ids.insert(docid);
self.to_delete_docids.insert(docid);
}
pub fn delete_documents(&mut self, docids: &RoaringBitmap) {
self.documents_ids |= docids;
self.to_delete_docids |= docids;
}
pub fn delete_external_id(&mut self, external_id: &str) -> Option<u32> {
@@ -63,28 +64,30 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
Some(docid)
}
pub fn execute(self) -> Result<DocumentDeletionResult> {
pub fn execute(mut self) -> Result<DocumentDeletionResult> {
self.index.set_updated_at(self.wtxn, &OffsetDateTime::now_utc())?;
// We retrieve the current documents ids that are in the database.
let mut documents_ids = self.index.documents_ids(self.wtxn)?;
let mut soft_deleted_docids = self.index.soft_deleted_documents_ids(self.wtxn)?;
let current_documents_ids_len = documents_ids.len();
// We can and must stop removing documents in a database that is empty.
if documents_ids.is_empty() {
return Ok(DocumentDeletionResult {
deleted_documents: 0,
remaining_documents: current_documents_ids_len,
});
// but if there was still documents to delete we clear the database entirely
if !soft_deleted_docids.is_empty() {
ClearDocuments::new(self.wtxn, self.index).execute()?;
}
return Ok(DocumentDeletionResult { deleted_documents: 0, remaining_documents: 0 });
}
// We remove the documents ids that we want to delete
// from the documents in the database and write them back.
documents_ids -= &self.documents_ids;
documents_ids -= &self.to_delete_docids;
self.index.put_documents_ids(self.wtxn, &documents_ids)?;
// We can execute a ClearDocuments operation when the number of documents
// to delete is exactly the number of documents in the database.
if current_documents_ids_len == self.documents_ids.len() {
if current_documents_ids_len == self.to_delete_docids.len() {
let remaining_documents = ClearDocuments::new(self.wtxn, self.index).execute()?;
return Ok(DocumentDeletionResult {
deleted_documents: current_documents_ids_len,
@@ -93,6 +96,50 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
}
let fields_ids_map = self.index.fields_ids_map(self.wtxn)?;
let mut field_distribution = self.index.field_distribution(self.wtxn)?;
// we update the field distribution
for docid in self.to_delete_docids.iter() {
let key = BEU32::new(docid);
let document =
self.index.documents.get(self.wtxn, &key)?.ok_or(
InternalError::DatabaseMissingEntry { db_name: "documents", key: None },
)?;
for (fid, _value) in document.iter() {
let field_name =
fields_ids_map.name(fid).ok_or(FieldIdMapMissingEntry::FieldId {
field_id: fid,
process: "delete documents",
})?;
if let Entry::Occupied(mut entry) = field_distribution.entry(field_name.to_string())
{
match entry.get().checked_sub(1) {
Some(0) | None => entry.remove(),
Some(count) => entry.insert(count),
};
}
}
}
self.index.put_field_distribution(self.wtxn, &field_distribution)?;
soft_deleted_docids |= &self.to_delete_docids;
// if we have less documents to delete than the threshold we simply save them in
// the `soft_deleted_documents_ids` bitmap and early exit.
if soft_deleted_docids.len() < DELETE_DOCUMENTS_THRESHOLD {
self.index.put_soft_deleted_documents_ids(self.wtxn, &soft_deleted_docids)?;
return Ok(DocumentDeletionResult {
deleted_documents: self.to_delete_docids.len(),
remaining_documents: documents_ids.len(),
});
}
// There is more than documents to delete than the threshold we needs to delete them all
self.to_delete_docids = soft_deleted_docids;
// and we can reset the soft deleted bitmap
self.index.put_soft_deleted_documents_ids(self.wtxn, &RoaringBitmap::new())?;
let primary_key = self.index.primary_key(self.wtxn)?.ok_or_else(|| {
InternalError::DatabaseMissingEntry {
db_name: db_name::MAIN,
@@ -127,23 +174,16 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
documents,
} = self.index;
// Number of fields for each document that has been deleted.
let mut fields_ids_distribution_diff = HashMap::new();
// Retrieve the words and the external documents ids contained in the documents.
let mut words = Vec::new();
let mut external_ids = Vec::new();
for docid in &self.documents_ids {
for docid in &self.to_delete_docids {
// We create an iterator to be able to get the content and delete the document
// content itself. It's faster to acquire a cursor to get and delete,
// as we avoid traversing the LMDB B-Tree two times but only once.
let key = BEU32::new(docid);
let mut iter = documents.range_mut(self.wtxn, &(key..=key))?;
if let Some((_key, obkv)) = iter.next().transpose()? {
for (field_id, _) in obkv.iter() {
*fields_ids_distribution_diff.entry(field_id).or_default() += 1;
}
if let Some(content) = obkv.get(id_field) {
let external_id = match serde_json::from_slice(content).unwrap() {
Value::String(string) => SmallString32::from(string.as_str()),
@@ -171,24 +211,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
}
}
let mut field_distribution = self.index.field_distribution(self.wtxn)?;
// We use pre-calculated number of fields occurrences that needs to be deleted
// to reflect deleted documents.
// If all field occurrences are removed, delete the entry from distribution.
// Otherwise, insert new number of occurrences (current_count - count_diff).
for (field_id, count_diff) in fields_ids_distribution_diff {
let field_name = fields_ids_map.name(field_id).unwrap();
if let Entry::Occupied(mut entry) = field_distribution.entry(field_name.to_string()) {
match entry.get().checked_sub(count_diff) {
Some(0) | None => entry.remove(),
Some(count) => entry.insert(count),
};
}
}
self.index.put_field_distribution(self.wtxn, &field_distribution)?;
// We create the FST map of the external ids that we must delete.
external_ids.sort_unstable();
let external_ids_to_delete = fst::Set::from_iter(external_ids)?;
@@ -214,7 +236,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
word_docids,
word.as_str(),
must_remove,
&self.documents_ids,
&self.to_delete_docids,
)?;
remove_from_word_docids(
@@ -222,7 +244,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
exact_word_docids,
word.as_str(),
must_remove,
&self.documents_ids,
&self.to_delete_docids,
)?;
}
@@ -256,12 +278,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
self.index.put_words_fst(self.wtxn, &new_words_fst)?;
let prefixes_to_delete =
remove_from_word_prefix_docids(self.wtxn, word_prefix_docids, &self.documents_ids)?;
remove_from_word_prefix_docids(self.wtxn, word_prefix_docids, &self.to_delete_docids)?;
let exact_prefix_to_delete = remove_from_word_prefix_docids(
self.wtxn,
exact_word_prefix_docids,
&self.documents_ids,
&self.to_delete_docids,
)?;
let all_prefixes_to_delete = prefixes_to_delete.op().add(&exact_prefix_to_delete).union();
@@ -293,7 +315,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
while let Some(result) = iter.next() {
let (key, mut docids) = result?;
let previous_len = docids.len();
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
if docids.is_empty() {
// safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
@@ -314,7 +336,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
while let Some(result) = iter.next() {
let (bytes, mut docids) = result?;
let previous_len = docids.len();
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
if docids.is_empty() {
// safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
@@ -332,7 +354,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
while let Some(result) = iter.next() {
let (bytes, mut docids) = result?;
let previous_len = docids.len();
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
if docids.is_empty() {
// safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
@@ -351,7 +373,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
while let Some(result) = iter.next() {
let (bytes, mut docids) = result?;
let previous_len = docids.len();
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
if docids.is_empty() {
// safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
@@ -368,7 +390,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let mut iter = field_id_word_count_docids.iter_mut(self.wtxn)?;
while let Some((key, mut docids)) = iter.next().transpose()? {
let previous_len = docids.len();
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
if docids.is_empty() {
// safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
@@ -386,7 +408,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let (points_to_remove, docids_to_remove): (Vec<_>, RoaringBitmap) = rtree
.iter()
.filter(|&point| self.documents_ids.contains(point.data.0))
.filter(|&point| self.to_delete_docids.contains(point.data.0))
.cloned()
.map(|point| (point, point.data.0))
.unzip();
@@ -403,46 +425,46 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
remove_docids_from_facet_field_id_number_docids(
self.wtxn,
facet_id_f64_docids,
&self.documents_ids,
&self.to_delete_docids,
)?;
remove_docids_from_facet_field_id_string_docids(
self.wtxn,
facet_id_string_docids,
&self.documents_ids,
&self.to_delete_docids,
)?;
// Remove the documents ids from the faceted documents ids.
for field_id in self.index.faceted_fields_ids(self.wtxn)? {
// Remove docids from the number faceted documents ids
let mut docids = self.index.number_faceted_documents_ids(self.wtxn, field_id)?;
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
self.index.put_number_faceted_documents_ids(self.wtxn, field_id, &docids)?;
remove_docids_from_field_id_docid_facet_value(
self.wtxn,
field_id_docid_facet_f64s,
field_id,
&self.documents_ids,
&self.to_delete_docids,
|(_fid, docid, _value)| docid,
)?;
// Remove docids from the string faceted documents ids
let mut docids = self.index.string_faceted_documents_ids(self.wtxn, field_id)?;
docids -= &self.documents_ids;
docids -= &self.to_delete_docids;
self.index.put_string_faceted_documents_ids(self.wtxn, field_id, &docids)?;
remove_docids_from_field_id_docid_facet_value(
self.wtxn,
field_id_docid_facet_strings,
field_id,
&self.documents_ids,
&self.to_delete_docids,
|(_fid, docid, _value)| docid,
)?;
}
Ok(DocumentDeletionResult {
deleted_documents: self.documents_ids.len(),
deleted_documents: self.to_delete_docids.len(),
remaining_documents: documents_ids.len(),
})
}
@@ -741,26 +763,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
{"docid":"1_5","label":"letter"},
{"docid":"1_7","label":"abstract,cartoon,design,pattern"},
{"docid":"1_36","label":"drawing,painting,pattern"},
{"docid":"1_37","label":"art,drawing,outdoor"},
{"docid":"1_38","label":"aquarium,art,drawing"},
{"docid":"1_39","label":"abstract"},
{"docid":"1_40","label":"cartoon"},
{"docid":"1_41","label":"art,drawing"},
{"docid":"1_42","label":"art,pattern"},
{"docid":"1_43","label":"abstract,art,drawing,pattern"},
{"docid":"1_44","label":"drawing"},
{"docid":"1_45","label":"art"},
{"docid":"1_46","label":"abstract,colorfulness,pattern"},
{"docid":"1_47","label":"abstract,pattern"},
{"docid":"1_52","label":"abstract,cartoon"},
{"docid":"1_57","label":"abstract,drawing,pattern"},
{"docid":"1_58","label":"abstract,art,cartoon"},
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_39", "label": "abstract" },
{ "docid": "1_40", "label": "cartoon" },
{ "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_44", "label": "drawing" },
{ "docid": "1_45", "label": "art" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
insert_documents(&mut wtxn, &index, content);
@@ -788,26 +810,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
{"docid":"1_5","label":"letter"},
{"docid":"1_7","label":"abstract,cartoon,design,pattern"},
{"docid":"1_36","label":"drawing,painting,pattern"},
{"docid":"1_37","label":"art,drawing,outdoor"},
{"docid":"1_38","label":"aquarium,art,drawing"},
{"docid":"1_39","label":"abstract"},
{"docid":"1_40","label":"cartoon"},
{"docid":"1_41","label":"art,drawing"},
{"docid":"1_42","label":"art,pattern"},
{"docid":"1_43","label":"abstract,art,drawing,pattern"},
{"docid":"1_44","label":"drawing"},
{"docid":"1_45","label":"art"},
{"docid":"1_46","label":"abstract,colorfulness,pattern"},
{"docid":"1_47","label":"abstract,pattern"},
{"docid":"1_52","label":"abstract,cartoon"},
{"docid":"1_57","label":"abstract,drawing,pattern"},
{"docid":"1_58","label":"abstract,art,cartoon"},
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_39", "label": "abstract" },
{ "docid": "1_40", "label": "cartoon" },
{ "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_44", "label": "drawing" },
{ "docid": "1_45", "label": "art" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
insert_documents(&mut wtxn, &index, content);
@@ -841,26 +863,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
{"docid":"1_5","label":"letter"},
{"docid":"1_7","label":"abstract,cartoon,design,pattern"},
{"docid":"1_36","label":"drawing,painting,pattern"},
{"docid":"1_37","label":"art,drawing,outdoor"},
{"docid":"1_38","label":"aquarium,art,drawing"},
{"docid":"1_39","label":"abstract"},
{"docid":"1_40","label":"cartoon"},
{"docid":"1_41","label":"art,drawing"},
{"docid":"1_42","label":"art,pattern"},
{"docid":"1_43","label":"abstract,art,drawing,pattern"},
{"docid":"1_44","label":"drawing"},
{"docid":"1_45","label":"art"},
{"docid":"1_46","label":"abstract,colorfulness,pattern"},
{"docid":"1_47","label":"abstract,pattern"},
{"docid":"1_52","label":"abstract,cartoon"},
{"docid":"1_57","label":"abstract,drawing,pattern"},
{"docid":"1_58","label":"abstract,art,cartoon"},
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
{"docid": "1_4", "label": "sign"},
{"docid": "1_5", "label": "letter"},
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
{"docid": "1_36","label": "drawing,painting,pattern"},
{"docid": "1_37","label": "art,drawing,outdoor"},
{"docid": "1_38","label": "aquarium,art,drawing"},
{"docid": "1_39","label": "abstract"},
{"docid": "1_40","label": "cartoon"},
{"docid": "1_41","label": "art,drawing"},
{"docid": "1_42","label": "art,pattern"},
{"docid": "1_43","label": "abstract,art,drawing,pattern"},
{"docid": "1_44","label": "drawing"},
{"docid": "1_45","label": "art"},
{"docid": "1_46","label": "abstract,colorfulness,pattern"},
{"docid": "1_47","label": "abstract,pattern"},
{"docid": "1_52","label": "abstract,cartoon"},
{"docid": "1_57","label": "abstract,drawing,pattern"},
{"docid": "1_58","label": "abstract,art,cartoon"},
{"docid": "1_68","label": "design"},
{"docid": "1_69","label": "geometry"}
]);
insert_documents(&mut wtxn, &index, content);
@@ -896,26 +918,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"id":"1","city":"Lille", "_geo": { "lat": 50.6299 as f32, "lng": 3.0569 as f32 } },
{"id":"2","city":"Mons-en-Barœul", "_geo": { "lat": 50.6415 as f32, "lng": 3.1106 as f32 } },
{"id":"3","city":"Hellemmes", "_geo": { "lat": 50.6312 as f32, "lng": 3.1106 as f32 } },
{"id":"4","city":"Villeneuve-d'Ascq", "_geo": { "lat": 50.6224 as f32, "lng": 3.1476 as f32 } },
{"id":"5","city":"Hem", "_geo": { "lat": 50.6552 as f32, "lng": 3.1897 as f32 } },
{"id":"6","city":"Roubaix", "_geo": { "lat": 50.6924 as f32, "lng": 3.1763 as f32 } },
{"id":"7","city":"Tourcoing", "_geo": { "lat": 50.7263 as f32, "lng": 3.1541 as f32 } },
{"id":"8","city":"Mouscron", "_geo": { "lat": 50.7453 as f32, "lng": 3.2206 as f32 } },
{"id":"9","city":"Tournai", "_geo": { "lat": 50.6053 as f32, "lng": 3.3758 as f32 } },
{"id":"10","city":"Ghent", "_geo": { "lat": 51.0537 as f32, "lng": 3.6957 as f32 } },
{"id":"11","city":"Brussels", "_geo": { "lat": 50.8466 as f32, "lng": 4.3370 as f32 } },
{"id":"12","city":"Charleroi", "_geo": { "lat": 50.4095 as f32, "lng": 4.4347 as f32 } },
{"id":"13","city":"Mons", "_geo": { "lat": 50.4502 as f32, "lng": 3.9623 as f32 } },
{"id":"14","city":"Valenciennes", "_geo": { "lat": 50.3518 as f32, "lng": 3.5326 as f32 } },
{"id":"15","city":"Arras", "_geo": { "lat": 50.2844 as f32, "lng": 2.7637 as f32 } },
{"id":"16","city":"Cambrai", "_geo": { "lat": 50.1793 as f32, "lng": 3.2189 as f32 } },
{"id":"17","city":"Bapaume", "_geo": { "lat": 50.1112 as f32, "lng": 2.8547 as f32 } },
{"id":"18","city":"Amiens", "_geo": { "lat": 49.9314 as f32, "lng": 2.2710 as f32 } },
{"id":"19","city":"Compiègne", "_geo": { "lat": 49.4449 as f32, "lng": 2.7913 as f32 } },
{"id":"20","city":"Paris", "_geo": { "lat": 48.9021 as f32, "lng": 2.3708 as f32 } }
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } },
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } },
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } },
{ "id": "4", "city": "Villeneuve-d'Ascq", "_geo": { "lat": 50.6224, "lng": 3.1476 } },
{ "id": "5", "city": "Hem", "_geo": { "lat": 50.6552, "lng": 3.1897 } },
{ "id": "6", "city": "Roubaix", "_geo": { "lat": 50.6924, "lng": 3.1763 } },
{ "id": "7", "city": "Tourcoing", "_geo": { "lat": 50.7263, "lng": 3.1541 } },
{ "id": "8", "city": "Mouscron", "_geo": { "lat": 50.7453, "lng": 3.2206 } },
{ "id": "9", "city": "Tournai", "_geo": { "lat": 50.6053, "lng": 3.3758 } },
{ "id": "10", "city": "Ghent", "_geo": { "lat": 51.0537, "lng": 3.6957 } },
{ "id": "11", "city": "Brussels", "_geo": { "lat": 50.8466, "lng": 4.3370 } },
{ "id": "12", "city": "Charleroi", "_geo": { "lat": 50.4095, "lng": 4.4347 } },
{ "id": "13", "city": "Mons", "_geo": { "lat": 50.4502, "lng": 3.9623 } },
{ "id": "14", "city": "Valenciennes", "_geo": { "lat": 50.3518, "lng": 3.5326 } },
{ "id": "15", "city": "Arras", "_geo": { "lat": 50.2844, "lng": 2.7637 } },
{ "id": "16", "city": "Cambrai", "_geo": { "lat": 50.1793, "lng": 3.2189 } },
{ "id": "17", "city": "Bapaume", "_geo": { "lat": 50.1112, "lng": 2.8547 } },
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } },
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } },
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } }
]);
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
@@ -951,26 +973,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
{"docid":"1_5","label":"letter"},
{"docid":"1_7","label":"abstract,cartoon,design,pattern"},
{"docid":"1_36","label":"drawing,painting,pattern"},
{"docid":"1_37","label":"art,drawing,outdoor"},
{"docid":"1_38","label":"aquarium,art,drawing"},
{"docid":"1_39","label":"abstract"},
{"docid":"1_40","label":"cartoon"},
{"docid":"1_41","label":"art,drawing"},
{"docid":"1_42","label":"art,pattern"},
{"docid":"1_43","label":"abstract,art,drawing,pattern"},
{"docid":"1_44","label":"drawing"},
{"docid":"1_45","label":"art"},
{"docid":"1_46","label":"abstract,colorfulness,pattern"},
{"docid":"1_47","label":"abstract,pattern"},
{"docid":"1_52","label":"abstract,cartoon"},
{"docid":"1_57","label":"abstract,drawing,pattern"},
{"docid":"1_58","label":"abstract,art,cartoon"},
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_39", "label": "abstract" },
{ "docid": "1_40", "label": "cartoon" },
{ "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_44", "label": "drawing" },
{ "docid": "1_45", "label": "art" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
insert_documents(&mut wtxn, &index, content);
@@ -1021,26 +1043,26 @@ mod tests {
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
{"docid":"1_5","label":"letter"},
{"docid":"1_7","label":"abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
{"docid":"1_36","label":"drawing,painting,pattern"},
{"docid":"1_37","label":"art,drawing,outdoor"},
{"docid":"1_38","label":"aquarium,art,drawing", "title": "Nemo"},
{"docid":"1_39","label":"abstract"},
{"docid":"1_40","label":"cartoon"},
{"docid":"1_41","label":"art,drawing"},
{"docid":"1_42","label":"art,pattern"},
{"docid":"1_43","label":"abstract,art,drawing,pattern", "number": 32i32},
{"docid":"1_44","label":"drawing", "number": 44i32},
{"docid":"1_45","label":"art"},
{"docid":"1_46","label":"abstract,colorfulness,pattern"},
{"docid":"1_47","label":"abstract,pattern"},
{"docid":"1_52","label":"abstract,cartoon"},
{"docid":"1_57","label":"abstract,drawing,pattern"},
{"docid":"1_58","label":"abstract,art,cartoon"},
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
{ "docid": "1_4", "label": "sign"},
{ "docid": "1_5", "label": "letter"},
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
{ "docid": "1_36", "label": "drawing,painting,pattern"},
{ "docid": "1_37", "label": "art,drawing,outdoor"},
{ "docid": "1_38", "label": "aquarium,art,drawing", "title": "Nemo"},
{ "docid": "1_39", "label": "abstract"},
{ "docid": "1_40", "label": "cartoon"},
{ "docid": "1_41", "label": "art,drawing"},
{ "docid": "1_42", "label": "art,pattern"},
{ "docid": "1_43", "label": "abstract,art,drawing,pattern", "number": 32i32},
{ "docid": "1_44", "label": "drawing", "number": 44i32},
{ "docid": "1_45", "label": "art"},
{ "docid": "1_46", "label": "abstract,colorfulness,pattern"},
{ "docid": "1_47", "label": "abstract,pattern"},
{ "docid": "1_52", "label": "abstract,cartoon"},
{ "docid": "1_57", "label": "abstract,drawing,pattern"},
{ "docid": "1_58", "label": "abstract,art,cartoon"},
{ "docid": "1_68", "label": "design"},
{ "docid": "1_69", "label": "geometry"}
]);
insert_documents(&mut wtxn, &index, content);