snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-10-31 07:56:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix the indexation tests

Browse Source
This commit is contained in:
Kerollmops
2022-06-15 14:35:19 +02:00
parent fcfc4caf8c
commit 399eec5c01
15 changed files with 288 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
benchmarks/benches/indexing.rs
View File

@@ -132,12 +132,13 @@ fn indexing_songs_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -223,11 +224,12 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -279,11 +281,12 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_1_2, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS_1_2, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -294,19 +297,21 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_3_4, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS_3_4, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS_4_4, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS_4_4, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -339,13 +344,14 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -377,12 +383,13 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -415,12 +422,13 @@ fn indexing_wiki(c: &mut Criterion) {
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -507,11 +515,12 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv"); let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -564,12 +573,13 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_1_2, "csv"); utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_1_2, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -581,24 +591,26 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_3_4, "csv"); utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_3_4, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = let documents =
utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_4_4, "csv"); utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES_4_4, "csv");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -631,12 +643,13 @@ fn indexing_movies_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::MOVIES, "json"); let documents = utils::documents_from(datasets_paths::MOVIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -720,11 +733,12 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::MOVIES, "json"); let documents = utils::documents_from(datasets_paths::MOVIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -775,12 +789,13 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
// as we don't care about the time it takes. // as we don't care about the time it takes.
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::MOVIES_1_2, "json"); let documents = utils::documents_from(datasets_paths::MOVIES_1_2, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -791,21 +806,23 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::MOVIES_3_4, "json"); let documents = utils::documents_from(datasets_paths::MOVIES_3_4, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::MOVIES_4_4, "json"); let documents = utils::documents_from(datasets_paths::MOVIES_4_4, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -861,12 +878,13 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json"); let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -922,11 +940,12 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json"); let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -984,12 +1003,13 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json"); let documents = utils::documents_from(datasets_paths::NESTED_MOVIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1021,12 +1041,13 @@ fn indexing_geo(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "jsonl"); let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "jsonl");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1113,11 +1134,12 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ())
.unwrap(); .unwrap();
let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "json"); let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "json");
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

7
cli/src/main.rs
View File

@@ -255,7 +255,7 @@ impl Performer for DocumentAddition {
let bar = progesses.add(bar); let bar = progesses.add(bar);
bars.push(bar); bars.push(bar);
} }
let mut addition = milli::update::IndexDocuments::new( let addition = milli::update::IndexDocuments::new(
&mut txn, &mut txn,
&index, &index,
&config, &config,
@@ -263,7 +263,10 @@ impl Performer for DocumentAddition {
|step| indexing_callback(step, &bars), |step| indexing_callback(step, &bars),
) )
.unwrap(); .unwrap();
addition.add_documents(reader)?; let (addition, user_error) = addition.add_documents(reader)?;
if let Err(error) = user_error {
return Err(error.into());
}
std::thread::spawn(move || { std::thread::spawn(move || {
progesses.join().unwrap(); progesses.join().unwrap();

6
http-ui/src/main.rs
View File

@@ -374,7 +374,7 @@ async fn main() -> anyhow::Result<()> {
}); });
}; };
let mut builder = milli::update::IndexDocuments::new( let builder = milli::update::IndexDocuments::new(
&mut wtxn, &mut wtxn,
&index_cloned, &index_cloned,
GLOBAL_CONFIG.get().unwrap(), GLOBAL_CONFIG.get().unwrap(),
@@ -397,8 +397,8 @@ async fn main() -> anyhow::Result<()> {
let documents = DocumentsBatchReader::from_reader(Cursor::new(documents))?; let documents = DocumentsBatchReader::from_reader(Cursor::new(documents))?;
builder.add_documents(documents)?; let (builder, user_error) = builder.add_documents(documents)?;
let _count = user_error?;
let result = builder.execute(); let result = builder.execute();
match result { match result {

25
milli/src/index.rs
View File

@@ -1212,10 +1212,11 @@ pub(crate) mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1234,7 +1235,7 @@ pub(crate) mod tests {
// we add all the documents a second time. we are supposed to get the same // we add all the documents a second time. we are supposed to get the same
// field_distribution in the end // field_distribution in the end
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
let content = documents!([ let content = documents!([
@@ -1242,7 +1243,8 @@ pub(crate) mod tests {
{ "id": 2, "name": "bob", "age": 20 }, { "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 }, { "id": 2, "name": "bob", "age": 20 },
]); ]);
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1265,10 +1267,11 @@ pub(crate) mod tests {
]); ]);
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1333,10 +1336,11 @@ pub(crate) mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1390,10 +1394,11 @@ pub(crate) mod tests {
]); ]);
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

5
milli/src/search/distinct/mod.rs
View File

@@ -97,14 +97,15 @@ mod test {
update_method: IndexDocumentsMethod::ReplaceDocuments, update_method: IndexDocumentsMethod::ReplaceDocuments,
..Default::default() ..Default::default()
}; };
let mut addition = let addition =
IndexDocuments::new(&mut txn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut txn, &index, &config, indexing_config, |_| ()).unwrap();
let reader = let reader =
crate::documents::DocumentsBatchReader::from_reader(Cursor::new(JSON.as_slice())) crate::documents::DocumentsBatchReader::from_reader(Cursor::new(JSON.as_slice()))
.unwrap(); .unwrap();
addition.add_documents(reader).unwrap(); let (addition, user_error) = addition.add_documents(reader).unwrap();
user_error.unwrap();
addition.execute().unwrap(); addition.execute().unwrap();
let fields_map = index.fields_ids_map(&txn).unwrap(); let fields_map = index.fields_ids_map(&txn).unwrap();

5
milli/src/search/facet/filter.rs
View File

@@ -648,10 +648,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

5
milli/src/update/clear_documents.rs
View File

@@ -100,9 +100,10 @@ mod tests {
]); ]);
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// Clear all documents from the database. // Clear all documents from the database.

18
milli/src/update/delete_documents.rs
View File

@@ -657,13 +657,13 @@ mod tests {
fn insert_documents<'t, R: std::io::Read + std::io::Seek>( fn insert_documents<'t, R: std::io::Read + std::io::Seek>(
wtxn: &mut RwTxn<'t, '_>, wtxn: &mut RwTxn<'t, '_>,
index: &'t Index, index: &'t Index,
documents: crate::documents::DocumentBatchReader<R>, documents: crate::documents::DocumentsBatchReader<R>,
) { ) {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap();
IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
builder.add_documents(documents).unwrap(); user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
} }
@@ -701,9 +701,10 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// delete those documents, ids are synchronous therefore 0, 1, and 2. // delete those documents, ids are synchronous therefore 0, 1, and 2.
@@ -736,9 +737,10 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// Delete not all of the documents but some of them. // Delete not all of the documents but some of them.

205
milli/src/update/index_documents/mod.rs
View File

@@ -141,7 +141,12 @@ where
// We check for user errors in this validator and if there is one, we can return // We check for user errors in this validator and if there is one, we can return
// the `IndexDocument` struct as it is valid to send more documents into it. // the `IndexDocument` struct as it is valid to send more documents into it.
// However, if there is an internal error we throw it away! // However, if there is an internal error we throw it away!
let reader = match validate_documents_batch(self.wtxn, self.index, reader)? { let reader = match validate_documents_batch(
self.wtxn,
self.index,
self.config.autogenerate_docids,
reader,
)? {
Ok(reader) => reader, Ok(reader) => reader,
Err(user_error) => return Ok((self, Err(user_error))), Err(user_error) => return Ok((self, Err(user_error))),
}; };
@@ -626,10 +631,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -642,10 +648,11 @@ mod tests {
// Second we send 1 document with id 1, to erase the previous ones. // Second we send 1 document with id 1, to erase the previous ones.
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "id": 1, "name": "updated kevin" } ]); let content = documents!([ { "id": 1, "name": "updated kevin" } ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -662,9 +669,11 @@ mod tests {
{ "id": 2, "name": "updated kevina" }, { "id": 2, "name": "updated kevina" },
{ "id": 3, "name": "updated benoit" } { "id": 3, "name": "updated benoit" }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
// Check that there is **always** 3 documents. // Check that there is **always** 3 documents.
@@ -694,10 +703,11 @@ mod tests {
update_method: IndexDocumentsMethod::UpdateDocuments, update_method: IndexDocumentsMethod::UpdateDocuments,
..Default::default() ..Default::default()
}; };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -722,9 +732,10 @@ mod tests {
// Second we send 1 document with id 1, to force it to be merged with the previous one. // Second we send 1 document with id 1, to force it to be merged with the previous one.
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "id": 1, "age": 25 } ]); let content = documents!([ { "id": 1, "age": 25 } ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -765,7 +776,7 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
assert!(builder.add_documents(content).is_err()); assert!(builder.add_documents(content).is_err());
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -794,10 +805,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -814,9 +826,10 @@ mod tests {
// Second we send 1 document with the generated uuid, to erase the previous ones. // Second we send 1 document with the generated uuid, to erase the previous ones.
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "name": "updated kevin", "id": kevin_uuid } ]); let content = documents!([ { "name": "updated kevin", "id": kevin_uuid } ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -856,9 +869,10 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -873,9 +887,10 @@ mod tests {
let content = documents!([ { "name": "new kevin" } ]); let content = documents!([ { "name": "new kevin" } ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -898,9 +913,10 @@ mod tests {
let content = documents!([]); let content = documents!([]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -924,7 +940,7 @@ mod tests {
let content = documents!([ { "id": "brume bleue", "name": "kevin" } ]); let content = documents!([ { "id": "brume bleue", "name": "kevin" } ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
assert!(builder.add_documents(content).is_err()); assert!(builder.add_documents(content).is_err());
@@ -934,9 +950,10 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
// There is a space in the document id. // There is a space in the document id.
let content = documents!([ { "id": 32, "name": "kevin" } ]); let content = documents!([ { "id": 32, "name": "kevin" } ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -963,9 +980,10 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1009,9 +1027,10 @@ mod tests {
update_method: IndexDocumentsMethod::ReplaceDocuments, update_method: IndexDocumentsMethod::ReplaceDocuments,
..Default::default() ..Default::default()
}; };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1020,7 +1039,7 @@ mod tests {
update_method: IndexDocumentsMethod::UpdateDocuments, update_method: IndexDocumentsMethod::UpdateDocuments,
..Default::default() ..Default::default()
}; };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let documents = documents!([ let documents = documents!([
{ {
@@ -1030,7 +1049,8 @@ mod tests {
} }
]); ]);
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
} }
@@ -1057,9 +1077,10 @@ mod tests {
update_method: IndexDocumentsMethod::ReplaceDocuments, update_method: IndexDocumentsMethod::ReplaceDocuments,
..Default::default() ..Default::default()
}; };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1099,10 +1120,11 @@ mod tests {
{ "id": 2, "_geo": { "lng": "42" }, "_geo.lat": "31" }, { "id": 2, "_geo": { "lng": "42" }, "_geo.lat": "31" },
{ "id": 3, "_geo.lat": 31, "_geo.lng": "42" }, { "id": 3, "_geo.lat": 31, "_geo.lng": "42" },
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1138,10 +1160,11 @@ mod tests {
let documents = documents!([ let documents = documents!([
{ "id": 0, "_geo": { "lng": 42 } } { "id": 0, "_geo": { "lng": 42 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
let error = builder.execute().unwrap_err(); let error = builder.execute().unwrap_err();
assert_eq!( assert_eq!(
&error.to_string(), &error.to_string(),
@@ -1151,10 +1174,11 @@ mod tests {
let documents = documents!([ let documents = documents!([
{ "id": 0, "_geo": { "lat": 42 } } { "id": 0, "_geo": { "lat": 42 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
let error = builder.execute().unwrap_err(); let error = builder.execute().unwrap_err();
assert_eq!( assert_eq!(
&error.to_string(), &error.to_string(),
@@ -1164,10 +1188,11 @@ mod tests {
let documents = documents!([ let documents = documents!([
{ "id": 0, "_geo": { "lat": "lol", "lng": 42 } } { "id": 0, "_geo": { "lat": "lol", "lng": 42 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
let error = builder.execute().unwrap_err(); let error = builder.execute().unwrap_err();
assert_eq!( assert_eq!(
&error.to_string(), &error.to_string(),
@@ -1177,10 +1202,11 @@ mod tests {
let documents = documents!([ let documents = documents!([
{ "id": 0, "_geo": { "lat": [12, 13], "lng": 42 } } { "id": 0, "_geo": { "lat": [12, 13], "lng": 42 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
let error = builder.execute().unwrap_err(); let error = builder.execute().unwrap_err();
assert_eq!( assert_eq!(
&error.to_string(), &error.to_string(),
@@ -1190,10 +1216,11 @@ mod tests {
let documents = documents!([ let documents = documents!([
{ "id": 0, "_geo": { "lat": 12, "lng": "hello" } } { "id": 0, "_geo": { "lat": 12, "lng": "hello" } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(documents).unwrap(); let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
let error = builder.execute().unwrap_err(); let error = builder.execute().unwrap_err();
assert_eq!( assert_eq!(
&error.to_string(), &error.to_string(),
@@ -1217,10 +1244,11 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId")); assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId"));
@@ -1237,10 +1265,11 @@ mod tests {
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
let external_documents_ids = index.external_documents_ids(&wtxn).unwrap(); let external_documents_ids = index.external_documents_ids(&wtxn).unwrap();
assert!(external_documents_ids.get("30").is_some()); assert!(external_documents_ids.get("30").is_some());
@@ -1249,10 +1278,11 @@ mod tests {
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } { "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } }
]); ]);
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1281,10 +1311,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1315,10 +1346,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1373,10 +1405,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1426,10 +1459,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1558,10 +1592,11 @@ mod tests {
]); ]);
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1628,10 +1663,11 @@ mod tests {
// index the documents // index the documents
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1720,10 +1756,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1737,10 +1774,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1759,10 +1797,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1787,10 +1826,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1832,10 +1872,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
} }
@@ -1870,10 +1911,11 @@ mod tests {
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// Create one long document // Create one long document
@@ -1884,10 +1926,11 @@ mod tests {
// Index this one long document // Index this one long document
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1901,7 +1944,7 @@ mod tests {
let index = Index::new(options, tmp).unwrap(); let index = Index::new(options, tmp).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let indexer_config = IndexerConfig::default(); let indexer_config = IndexerConfig::default();
let mut builder = IndexDocuments::new( let builder = IndexDocuments::new(
&mut wtxn, &mut wtxn,
&index, &index,
&indexer_config, &indexer_config,
@@ -1930,8 +1973,10 @@ mod tests {
"branch_id_number": 0 "branch_id_number": 0
}]}; }]};
builder.add_documents(doc1).unwrap(); let (builder, user_error) = builder.add_documents(doc1).unwrap();
builder.add_documents(doc2).unwrap(); user_error.unwrap();
let (builder, user_error) = builder.add_documents(doc2).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();

1
milli/src/update/index_documents/validate.rs
View File

@@ -15,6 +15,7 @@ use crate::{Index, Result};
pub fn validate_documents_batch<R: Read + Seek>( pub fn validate_documents_batch<R: Read + Seek>(
rtxn: &heed::RoTxn, rtxn: &heed::RoTxn,
index: &Index, index: &Index,
autogenerate_docids: bool,
reader: DocumentsBatchReader<R>, reader: DocumentsBatchReader<R>,
) -> Result<StdResult<DocumentsBatchReader<R>, UserError>> { ) -> Result<StdResult<DocumentsBatchReader<R>, UserError>> {
let mut cursor = reader.into_cursor(); let mut cursor = reader.into_cursor();

70
milli/src/update/settings.rs
View File

@@ -735,10 +735,11 @@ mod tests {
]); ]);
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -798,10 +799,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -850,10 +852,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -880,10 +883,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// In the same transaction we change the displayed fields to be only the age. // In the same transaction we change the displayed fields to be only the age.
@@ -934,10 +938,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -974,10 +979,11 @@ mod tests {
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1016,10 +1022,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1067,10 +1074,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1110,10 +1118,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1142,10 +1151,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1172,10 +1182,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// In the same transaction we provide some stop_words // In the same transaction we provide some stop_words
@@ -1251,10 +1262,11 @@ mod tests {
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
// In the same transaction provide some synonyms // In the same transaction provide some synonyms
@@ -1389,10 +1401,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
@@ -1452,10 +1465,11 @@ mod tests {
]); ]);
let indexing_config = let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap(); .unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

6
milli/tests/search/facet_distribution.rs
View File

@@ -29,8 +29,7 @@ fn test_facet_distribution_with_no_facet_values() {
let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() }; let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() };
let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder = IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let mut documents_builder = DocumentsBatchBuilder::new(Vec::new()); let mut documents_builder = DocumentsBatchBuilder::new(Vec::new());
let reader = Cursor::new( let reader = Cursor::new(
r#"{ r#"{
@@ -55,7 +54,8 @@ fn test_facet_distribution_with_no_facet_values() {
// index documents // index documents
let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap(); let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

6
milli/tests/search/mod.rs
View File

@@ -61,8 +61,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() }; let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() };
let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder = IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let mut documents_builder = DocumentsBatchBuilder::new(Vec::new()); let mut documents_builder = DocumentsBatchBuilder::new(Vec::new());
let reader = Cursor::new(CONTENT.as_bytes()); let reader = Cursor::new(CONTENT.as_bytes());
@@ -75,7 +74,8 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
// index documents // index documents
let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap(); let content = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap();
builder.add_documents(content).unwrap(); let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

6
milli/tests/search/query_criteria.rs
View File

@@ -390,8 +390,7 @@ fn criteria_ascdesc() {
// index documents // index documents
let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() }; let config = IndexerConfig { max_memory: Some(10 * 1024 * 1024), ..Default::default() };
let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() }; let indexing_config = IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let mut builder = let builder = IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let mut batch_builder = DocumentsBatchBuilder::new(Vec::new()); let mut batch_builder = DocumentsBatchBuilder::new(Vec::new());
@@ -422,7 +421,8 @@ fn criteria_ascdesc() {
let vector = batch_builder.into_inner().unwrap(); let vector = batch_builder.into_inner().unwrap();
let reader = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap(); let reader = DocumentsBatchReader::from_reader(Cursor::new(vector)).unwrap();
builder.add_documents(reader).unwrap(); let (builder, user_error) = builder.add_documents(reader).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

7
milli/tests/search/typo_tolerance.rs
View File

@@ -127,11 +127,10 @@ fn test_typo_disabled_on_word() {
let mut txn = index.write_txn().unwrap(); let mut txn = index.write_txn().unwrap();
let config = IndexerConfig::default(); let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default(); let indexing_config = IndexDocumentsConfig::default();
let mut builder = let builder = IndexDocuments::new(&mut txn, &index, &config, indexing_config, |_| ()).unwrap();
IndexDocuments::new(&mut txn, &index, &config, indexing_config, |_| ()).unwrap();
builder.add_documents(documents).unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap(); builder.execute().unwrap();
txn.commit().unwrap(); txn.commit().unwrap();